wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-06-29 17:53:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

[projects] Wrap ProjectDefinition in a class (#5654)

Browse Source
This commit is contained in:
Stephanie Wang
2019-09-07 18:30:17 -07:00
committed by Philipp Moritz
parent d0125d4212
commit cb7102f31e
20 changed files with 161 additions and 144 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/ray/tests/test_projects.py
+36 -39
View File
@@ -20,61 +20,57 @@ if sys.version_info >= (3, 3):
else:
from mock import patch, DEFAULT
TEST_DIR = os.path.dirname(os.path.abspath(__file__))
TEST_DIR = os.path.join(
os.path.dirname(os.path.abspath(__file__)), "project_files")
def load_project_description(project_file):
path = os.path.join(TEST_DIR, "project_files", project_file)
path = os.path.join(TEST_DIR, project_file)
with open(path) as f:
return yaml.safe_load(f)
def test_validation_success():
project_files = [
"docker_project.yaml", "requirements_project.yaml",
"shell_project.yaml"
]
for project_file in project_files:
project_definition = load_project_description(project_file)
ray.projects.validate_project_schema(project_definition)
def test_validation():
project_dirs = ["docker_project", "requirements_project", "shell_project"]
for project_dir in project_dirs:
project_dir = os.path.join(TEST_DIR, project_dir)
ray.projects.ProjectDefinition(project_dir)
def test_validation_failure():
project_files = ["no_project1.yaml", "no_project2.yaml"]
for project_file in project_files:
project_definition = load_project_description(project_file)
bad_schema_dirs = ["no_project1"]
for project_dir in bad_schema_dirs:
project_dir = os.path.join(TEST_DIR, project_dir)
with pytest.raises(jsonschema.exceptions.ValidationError):
ray.projects.validate_project_schema(project_definition)
ray.projects.ProjectDefinition(project_dir)
def test_check_failure():
project_files = ["no_project3.yaml"]
for project_file in project_files:
project_definition = load_project_description(project_file)
bad_project_dirs = ["no_project2", "noproject3"]
for project_dir in bad_project_dirs:
project_dir = os.path.join(TEST_DIR, project_dir)
with pytest.raises(ValueError):
ray.projects.check_project_definition("", project_definition)
ray.projects.ProjectDefinition(project_dir)
def test_project_root():
path = os.path.join(TEST_DIR, "project_files", "project1")
assert ray.projects.find_root(path) == path
path = os.path.join(TEST_DIR, "project1")
project_definition = ray.projects.ProjectDefinition(path)
assert os.path.normpath(project_definition.root) == os.path.normpath(path)
path2 = os.path.join(TEST_DIR, "project_files", "project1", "subdir")
assert ray.projects.find_root(path2) == path
path2 = os.path.join(TEST_DIR, "project1", "subdir")
project_definition = ray.projects.ProjectDefinition(path2)
assert os.path.normpath(project_definition.root) == os.path.normpath(path)
path3 = "/tmp/"
assert ray.projects.find_root(path3) is None
with pytest.raises(ValueError):
project_definition = ray.projects.ProjectDefinition(path3)
def test_project_validation():
path = os.path.join(TEST_DIR, "project_files", "project1")
path = os.path.join(TEST_DIR, "project1")
subprocess.check_call(["ray", "project", "validate"], cwd=path)
def test_project_no_validation():
path = os.path.join(TEST_DIR, "project_files")
with pytest.raises(subprocess.CalledProcessError):
subprocess.check_call(["ray", "project", "validate"], cwd=path)
subprocess.check_call(["ray", "project", "validate"], cwd=TEST_DIR)
@contextmanager
@@ -89,7 +85,7 @@ def _chdir_and_back(d):
def run_test_project(project_dir, command, args):
# Run the CLI commands with patching
test_dir = os.path.join(TEST_DIR, "project_files", project_dir)
test_dir = os.path.join(TEST_DIR, project_dir)
with _chdir_and_back(test_dir):
runner = CliRunner()
with patch.multiple(
@@ -107,14 +103,14 @@ def test_session_start_default_project():
result, mock_calls, test_dir = run_test_project(
"session-tests/project-pass", start, [])
loaded_project = ray.projects.load_project(test_dir)
loaded_project = ray.projects.ProjectDefinition(test_dir)
assert result.exit_code == 0
# Part 1/3: Cluster Launching Call
create_or_update_cluster_call = mock_calls["create_or_update_cluster"]
assert create_or_update_cluster_call.call_count == 1
_, kwargs = create_or_update_cluster_call.call_args
assert kwargs["config_file"] == loaded_project["cluster"]
assert kwargs["config_file"] == loaded_project.cluster_yaml()
# Part 2/3: Rsync Calls
rsync_call = mock_calls["rsync"]
@@ -122,21 +118,22 @@ def test_session_start_default_project():
# requirements.txt.
assert rsync_call.call_count == 2
_, kwargs = rsync_call.call_args
assert kwargs["source"] == loaded_project["environment"]["requirements"]
assert kwargs["source"] == loaded_project.config["environment"][
"requirements"]
# Part 3/3: Exec Calls
exec_cluster_call = mock_calls["exec_cluster"]
commands_executed = []
for _, kwargs in exec_cluster_call.call_args_list:
commands_executed.append(kwargs["cmd"].replace(
"cd {}; ".format(loaded_project["name"]), ""))
"cd {}; ".format(loaded_project.working_directory()), ""))
expected_commands = loaded_project["environment"]["shell"]
expected_commands = loaded_project.config["environment"]["shell"]
expected_commands += [
command["command"] for command in loaded_project["commands"]
command["command"] for command in loaded_project.config["commands"]
]
if "requirements" in loaded_project["environment"]:
if "requirements" in loaded_project.config["environment"]:
assert any("pip install -r" for cmd in commands_executed)
# pop the `pip install` off commands executed
commands_executed = [
@@ -171,7 +168,7 @@ def test_session_create_command():
["first", "--a", "1", "--b", "2"])
# Verify the project can be loaded.
ray.projects.load_project(test_dir)
ray.projects.ProjectDefinition(test_dir)
assert result.exit_code == 0
exec_cluster_call = mock_calls["exec_cluster"]