Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion actions/pyrosettacluster/conda/environment.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,4 +7,4 @@ dependencies:
- pyrosetta
- pip
- pip:
- pyrosetta-distributed
- pyrosetta-distributed>=0.0.5
2 changes: 1 addition & 1 deletion actions/pyrosettacluster/pixi/pixi.toml
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ version = "1.0.0"
pyrosetta = "*"

[pypi-dependencies]
pyrosetta-distributed = "*"
pyrosetta-distributed = ">=0.0.5"

[feature.{py_feature}.dependencies]
python = "{py_version}"
Expand Down
1 change: 1 addition & 0 deletions actions/pyrosettacluster/uv/requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ cloudpickle>=1.5.0
cryptography>=2.8
dask>=2.16.0
dask-jobqueue>=0.7.0
decorator>=4.3.0
distributed>=2.16.0
gitpython>=3.1.1
jupyter>=1.0.0
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ cloudpickle>=1.5.0
cryptography>=2.8
dask>=2.16.0
dask-jobqueue>=0.7.0
decorator>=4.3.0
distributed>=2.16.0
gitpython>=3.1.1
jupyter>=1.0.0
Expand Down
2 changes: 1 addition & 1 deletion pyrosettacluster/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
<a name="creating-environments-for-pyrosettacluster"></a>
# 🏠 Creating Environments for *PyRosettaCluster*
The *PyRosettaCluster* framework supports running reproducible PyRosetta simulations from reproducible virtual environments created with [*Conda*](https://docs.conda.io/), [*Mamba*](https://mamba.readthedocs.io/), [*uv*](https://docs.astral.sh/uv/), and [*Pixi*](https://pixi.sh/) environment managers. Please install [PyRosetta](https://www.pyrosetta.org/downloads) (with `cxx11thread.serialization` support) and the following packages to get started (and see the [envs](envs) directory for template environment configuration files)!
- `attrs`, `billiard`, `blosc`, `cloudpickle`, `cryptography`, `dask`, `dask-jobqueue`, `distributed`, `gitpython`, `numpy`, `pandas`, `python-xz`, `scipy`, `traitlets`
- `attrs`, `billiard`, `blosc`, `cloudpickle`, `cryptography`, `dask`, `dask-jobqueue`, `decorator`, `distributed`, `gitpython`, `numpy`, `pandas`, `python-xz`, `scipy`, `traitlets`

[Official Full List of Packages](https://github.com/RosettaCommons/rosetta/blob/main/tests/benchmark/tests/__init__.py#L69-L84)

Expand Down
12 changes: 12 additions & 0 deletions pyrosettacluster/dump_env_file.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@


import argparse
import json
import pyrosetta
import pyrosetta.distributed.io as io
import os
Expand Down Expand Up @@ -95,6 +96,7 @@ def main(
toml = metadata_kwargs.get("toml")
toml_format = metadata_kwargs.get("toml_format")
env_manager = metadata_kwargs.get("environment_manager") # may be `None` in legacy cases
env_manager_version = metadata_kwargs.get("environment_manager_version") # may be `None` in legacy cases

sha1 = instance_kwargs.get("sha1")
print("[INFO] " + "-" * 72)
Expand Down Expand Up @@ -145,6 +147,16 @@ def main(
)
write_file(env_file, environment)

# Maybe write environment manager version file; legacy fallback doesn't write
if env_manager is not None:
env_metadata_json_file = os.path.join(env_dir, "environment_metadata.json")
env_metadata_dict = {
"environment_manager": env_manager,
"environment_manager_version": env_manager_version,
}
env_metadata = json.dumps(env_metadata_dict, indent=2, sort_keys=True)
write_file(env_metadata_json_file, env_metadata)


def write_file(path: str, content: str) -> None:
"""Utility function for writing a file."""
Expand Down
82 changes: 82 additions & 0 deletions pyrosettacluster/recreate_env.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,10 @@


import argparse
import json
import os
import re
import shlex
import shutil
import subprocess
import tempfile
Expand Down Expand Up @@ -62,6 +64,32 @@ def run_subprocess(
raise RuntimeError(cmd) from ex


def get_env_manager_version(env_manager: str) -> str:
"""Return the version of the given environment manager."""

cmd = [env_manager, "--version"]
try:
output = subprocess.check_output(
cmd,
text=True,
stderr=subprocess.STDOUT,
).strip()
except (FileNotFoundError, subprocess.CalledProcessError):
version = ""
else:
found = re.search(r"v?(\d+(?:\.\d+)+)", output)
version = found.group(1) if found else ""

if not version:
cmd_str = shlex.join(cmd)
print(
f"[WARNING] Could not determine the {env_manager} environment manager "
f"version from running `{cmd_str}`."
)

return version


def uv_lock_package_present(lock_file: str, name: str) -> bool:
"""Test if a package name is specified in an input 'uv.lock' file."""

Expand All @@ -84,6 +112,60 @@ def recreate_environment(env_dir: str, env_manager: str, timeout: float, mirror_
The directory must already exist.
"""

env_metadata_json_file = os.path.join(env_dir, "environment_metadata.json")
if os.path.isfile(env_metadata_json_file):
try:
with open(env_metadata_json_file) as f:
env_metadata = json.load(f)
except (OSError, json.JSONDecodeError):
print(
"[WARNING] Could not read the `environment_metadata.json` file. "
"Skipping environment manager validation!"
)
else:
original_env_manager = env_metadata.get("environment_manager")
original_env_manager_version = env_metadata.get("environment_manager_version")
if env_manager != original_env_manager:
raise ValueError(
f"The environment manager used for the original `PyRosettaCluster` simulation was '{original_env_manager}', "
f"but the current environment manager is configured to be '{env_manager}'. Please run "
f"`export PYROSETTACLUSTER_ENVIRONMENT_MANAGER={original_env_manager}` or pass the "
f"`--env_manager {original_env_manager}` flag to continue."
)
if env_manager in ("pixi", "uv"):
env_manager_version = get_env_manager_version(env_manager)
if not env_manager_version and not original_env_manager_version:
print(
f"[WARNING] Skipping environment manager version check! The original {original_env_manager} version "
f"was not captured in the `PyRosettaCluster` full simulation record, and the currently installed {env_manager} "
f"version cannot be determined either. Please ensure that the {env_manager} version is identical to that "
"used to generate the original `PyRosettaCluster` result (if documented outside the full simulation record), "
"otherwise downstream `PyRosettaCluster` environment validation may fail even if the resolved dependencies "
"are equivalent due to differences in the generated lockfile format."
)
elif not env_manager_version and original_env_manager_version:
print(
f"[WARNING] Skipping environment manager version check! Please ensure that the currently installed {env_manager} "
f"version is version {original_env_manager_version}, otherwise downstream `PyRosettaCluster` environment validation"
"may fail even if the resolved dependencies are equivalent due to differences in the generated lockfile format."
)
elif env_manager_version and not original_env_manager_version:
print(
f"[WARNING] Skipping environment manager version check! The original {original_env_manager} version "
f"was not captured in the `PyRosettaCluster` full simulation record. Please ensure that the {env_manager} "
"version is identical to that used to generate the original `PyRosettaCluster` result (if documented "
"outside the full simulation record), otherwise downstream `PyRosettaCluster` environment validation may fail "
"even if the resolved dependencies are equivalent due to differences in the generated lockfile format."
)
elif env_manager_version != original_env_manager_version:
print(
f"[WARNING] The original {original_env_manager} version {original_env_manager_version} was used to generate "
f"the `PyRosettaCluster` result, but the currently installed {env_manager} version is {env_manager_version}. "
f"It is highly recommended to install {env_manager} version {original_env_manager_version} before continuing, "
"otherwise downstream `PyRosettaCluster` environment validation may fail even if the resolved dependencies are "
"equivalent due to differences in the generated lockfile format."
)

if env_manager == "pixi":
lock_file = os.path.join(env_dir, "pixi.lock")
if not os.path.isfile(lock_file):
Expand Down
Loading