From bdd7056394e818a990262c12743f924f6d015b2c Mon Sep 17 00:00:00 2001 From: c-bata Date: Fri, 4 Aug 2023 17:36:00 +0900 Subject: [PATCH] Replace ArtifactBackend with ArtifactStore --- optuna_dashboard/_app.py | 48 +++++++++++++++---- optuna_dashboard/artifact/_backend.py | 29 +++++++---- .../artifact/_backend_to_store.py | 32 +++++++++++++ optuna_dashboard/artifact/boto3.py | 10 ++++ optuna_dashboard/artifact/file_system.py | 11 +++++ optuna_dashboard/artifact/protocol.py | 14 ++++++ 6 files changed, 126 insertions(+), 18 deletions(-) create mode 100644 optuna_dashboard/artifact/_backend_to_store.py diff --git a/optuna_dashboard/_app.py b/optuna_dashboard/_app.py index 9b9de57d..12571829 100644 --- a/optuna_dashboard/_app.py +++ b/optuna_dashboard/_app.py @@ -7,6 +7,7 @@ import typing from typing import Any from typing import Optional from typing import Union +import warnings from bottle import Bottle from bottle import redirect @@ -36,10 +37,13 @@ from ._storage import get_trials from ._storage_url import get_storage from .artifact._backend import delete_all_artifacts from .artifact._backend import register_artifact_route +from .artifact._backend_to_store import ArtifactBackendToStore +from .artifact._backend_to_store import is_artifact_store if typing.TYPE_CHECKING: from _typeshed.wsgi import WSGIApplication + from optuna.artifacts._protocol import ArtifactStore from optuna_dashboard.artifact.protocol import ArtifactBackend @@ -54,7 +58,7 @@ cached_path_exists = functools.lru_cache(maxsize=10)(os.path.exists) def create_app( storage: BaseStorage, - artifact_backend: Optional[ArtifactBackend] = None, + artifact_store: Optional[ArtifactStore] = None, debug: bool = False, ) -> Bottle: app = Bottle() @@ -76,7 +80,7 @@ def create_app( @json_api_view def api_meta() -> dict[str, Any]: return { - "artifact_is_available": artifact_backend is not None, + "artifact_is_available": artifact_store is not None, } @app.get("/api/studies") @@ -156,9 +160,9 @@ def create_app( @app.delete("/api/studies/") @json_api_view def delete_study(study_id: int) -> dict[str, Any]: - if artifact_backend is not None: + if artifact_store is not None: system_attrs = storage.get_study_system_attrs(study_id) - delete_all_artifacts(artifact_backend, system_attrs) + delete_all_artifacts(artifact_store, system_attrs) try: storage.delete_study(study_id) @@ -347,7 +351,7 @@ def create_app( return static_file(filename, root=STATIC_DIR) register_rdb_migration_route(app, storage) - register_artifact_route(app, storage, artifact_backend) + register_artifact_route(app, storage, artifact_store) return app @@ -355,25 +359,49 @@ def run_server( storage: Union[str, BaseStorage], host: str = "localhost", port: int = 8080, - artifact_backend: Optional[ArtifactBackend] = None, + artifact_store: Optional[ArtifactStore | ArtifactBackend] = None, + *, + # TODO(c-bata): Remove this keyword argument in the v0.14.0 release. + artifact_backend: Optional[ArtifactBackend | ArtifactStore] = None, ) -> None: """Start running optuna-dashboard and blocks until the server terminates. This function uses wsgiref module which is not intended for the production use. If you want to run optuna-dashboard more secure and/or more fast, please use WSGI server like Gunicorn or uWSGI via :func:`wsgi` function. - - """ - app = create_app(get_storage(storage), artifact_backend=artifact_backend) + if artifact_backend is not None: + warnings.warn( + "The `artifact_backend` argument is deprecated. " + "Please use `artifact_store` instead.", + DeprecationWarning, + ) + artifact_store = ArtifactBackendToStore(artifact_backend) + if not is_artifact_store(artifact_store): + artifact_store = ArtifactBackendToStore(artifact_store) + + app = create_app(get_storage(storage), artifact_store=artifact_store) run(app, host=host, port=port) def wsgi( storage: Union[str, BaseStorage], + artifact_store: Optional[ArtifactBackend | ArtifactStore] = None, + *, + # TODO(c-bata): Remove this keyword argument in the v0.14.0 release. artifact_backend: Optional[ArtifactBackend] = None, ) -> WSGIApplication: """This function exposes WSGI interface for people who want to run on the production-class WSGI servers like Gunicorn or uWSGI. """ - return create_app(get_storage(storage), artifact_backend=artifact_backend) + if artifact_backend is not None: + warnings.warn( + "The `artifact_backend` argument is deprecated. " + "Please use `artifact_store` instead.", + DeprecationWarning, + ) + artifact_store = ArtifactBackendToStore(artifact_backend) + if not is_artifact_store(artifact_store): + artifact_store = ArtifactBackendToStore(artifact_store) + + return create_app(get_storage(storage), artifact_store=artifact_store) diff --git a/optuna_dashboard/artifact/_backend.py b/optuna_dashboard/artifact/_backend.py index 12f5390a..d90773d9 100644 --- a/optuna_dashboard/artifact/_backend.py +++ b/optuna_dashboard/artifact/_backend.py @@ -6,6 +6,7 @@ import mimetypes import os.path from typing import TYPE_CHECKING import uuid +import warnings from bottle import BaseRequest from bottle import Bottle @@ -24,6 +25,7 @@ if TYPE_CHECKING: from typing import Optional from typing import TypedDict + from optuna.artifacts._protocol import ArtifactStore from optuna.storages import BaseStorage from .protocol import ArtifactBackend @@ -57,11 +59,11 @@ def get_artifact_path( def register_artifact_route( - app: Bottle, storage: BaseStorage, artifact_backend: Optional[ArtifactBackend] + app: Bottle, storage: BaseStorage, artifact_store: Optional[ArtifactStore] ) -> None: @app.get("/artifacts///") def proxy_artifact(study_id: int, trial_id: int, artifact_id: str) -> HTTPResponse | bytes: - if artifact_backend is None: + if artifact_store is None: response.status = 400 # Bad Request return b"Cannot access to the artifacts." artifact_dict = _get_artifact_meta(storage, study_id, trial_id, artifact_id) @@ -73,13 +75,13 @@ def register_artifact_route( if encoding: headers["Content-Encodings"] = encoding - fp = artifact_backend.open(artifact_id) + fp = artifact_store.open_reader(artifact_id) return HTTPResponse(fp, headers=headers) @app.post("/api/artifacts//") @json_api_view def upload_artifact_api(study_id: int, trial_id: int) -> dict[str, Any]: - if artifact_backend is None: + if artifact_store is None: response.status = 400 # Bad Request return {"reason": "Cannot access to the artifacts."} file = request.json.get("file") @@ -90,7 +92,7 @@ def register_artifact_route( _, data = parse_data_uri(file) filename = request.json.get("filename", "") artifact_id = str(uuid.uuid4()) - artifact_backend.write(artifact_id, io.BytesIO(data)) + artifact_store.write(artifact_id, io.BytesIO(data)) mimetype, encoding = mimetypes.guess_type(filename) artifact = { @@ -115,10 +117,10 @@ def register_artifact_route( @app.delete("/api/artifacts///") @json_api_view def delete_artifact(study_id: int, trial_id: int, artifact_id: str) -> dict[str, Any]: - if artifact_backend is None: + if artifact_store is None: response.status = 400 # Bad Request return {"reason": "Cannot access to the artifacts."} - artifact_backend.remove(artifact_id) + artifact_store.remove(artifact_id) attr_key = _artifact_prefix(trial_id) + artifact_id storage.set_study_system_attr(study_id, attr_key, json.dumps(None)) @@ -134,7 +136,12 @@ def upload_artifact( mimetype: Optional[str] = None, encoding: Optional[str] = None, ) -> str: - """Upload an artifact (files), which is associated with the trial. + """[Deprecated] Upload an artifact (files), which is associated with the trial. + + .. note:: + + This function is deprecated. Please use `optuna.artifacts.upload_artifact + `_ instead. Example: .. code-block:: python @@ -151,6 +158,12 @@ def upload_artifact( upload_artifact(artifact_backend, trial, file_path) return ... """ + warnings.warn( + "This function is deprecated. Please use optuna.artifacts.upload_artifact() instead.\n" + "https://optuna.readthedocs.io/en/latest/reference/generated/optuna.artifacts.upload_artifact.html", + DeprecationWarning, + ) + filename = os.path.basename(file_path) storage = trial.storage trial_id = trial._trial_id diff --git a/optuna_dashboard/artifact/_backend_to_store.py b/optuna_dashboard/artifact/_backend_to_store.py new file mode 100644 index 00000000..7e496a5b --- /dev/null +++ b/optuna_dashboard/artifact/_backend_to_store.py @@ -0,0 +1,32 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from typing import BinaryIO + from typing import TypeGuard + + from optuna.artifacts._protocol import ArtifactStore + + from .protocol import ArtifactBackend + + +def is_artifact_store(store: ArtifactBackend | ArtifactStore) -> TypeGuard[ArtifactStore]: + return getattr(store, "open_reader") is not None + + +class ArtifactBackendToStore: + """Converts a Dashboard's ArtifactBackend to Optuna's ArtifactStore.""" + + def __init__(self, artifact_backend: ArtifactBackend) -> None: + self._backend = artifact_backend + + def open_reader(self, artifact_id: str) -> BinaryIO: + return self._backend.open(artifact_id) + + def write(self, artifact_id: str, content_body: BinaryIO) -> None: + self._backend.write(artifact_id, content_body) + + def remove(self, artifact_id: str) -> None: + self._backend.remove(artifact_id) diff --git a/optuna_dashboard/artifact/boto3.py b/optuna_dashboard/artifact/boto3.py index fcddf5d7..31b6a1b5 100644 --- a/optuna_dashboard/artifact/boto3.py +++ b/optuna_dashboard/artifact/boto3.py @@ -3,6 +3,7 @@ from __future__ import annotations import io import shutil from typing import TYPE_CHECKING +import warnings import boto3 from botocore.exceptions import ClientError @@ -19,6 +20,10 @@ if TYPE_CHECKING: class Boto3Backend: """An artifact backend for S3. + .. note:: + + This class is deprecated. Please use `optuna.artifacts.Boto3ArtifactStore `_ instead.", + Example: .. code-block:: python @@ -44,6 +49,11 @@ class Boto3Backend: # may close the source file object. # See https://github.com/boto/boto3/issues/929 self._avoid_buf_copy = avoid_buf_copy + warnings.warn( + "Boto3Backend is deprecated. Please use optuna.artifacts.Boto3ArtifactStore instead.\n" + "https://optuna.readthedocs.io/en/latest/reference/generated/optuna.artifacts.Boto3ArtifactStore.html", + DeprecationWarning, + ) def open(self, artifact_id: str) -> BinaryIO: try: diff --git a/optuna_dashboard/artifact/file_system.py b/optuna_dashboard/artifact/file_system.py index e714c86d..58c795e8 100644 --- a/optuna_dashboard/artifact/file_system.py +++ b/optuna_dashboard/artifact/file_system.py @@ -3,6 +3,7 @@ from __future__ import annotations import os import shutil from typing import TYPE_CHECKING +import warnings from optuna_dashboard.artifact.exceptions import ArtifactNotFound @@ -14,6 +15,11 @@ if TYPE_CHECKING: class FileSystemBackend: """An artifact backend for file systems. + .. note:: + + This class is deprecated. Please use `optuna.artifacts.FileSystemArtifactStore + `_ instead.", + Example: .. code-block:: python @@ -32,6 +38,11 @@ class FileSystemBackend: def __init__(self, base_path: str) -> None: self._base_path = base_path + warnings.warn( + "FileSystemBackend is deprecated. Please use optuna.artifacts.FileSystemArtifactStore instead.\n" + "https://optuna.readthedocs.io/en/latest/reference/generated/optuna.artifacts.FileSystemArtifactStore.html", + DeprecationWarning, + ) def open(self, artifact_id: str) -> BinaryIO: filepath = os.path.join(self._base_path, artifact_id) diff --git a/optuna_dashboard/artifact/protocol.py b/optuna_dashboard/artifact/protocol.py index 3f9ffdf2..b85fc82f 100644 --- a/optuna_dashboard/artifact/protocol.py +++ b/optuna_dashboard/artifact/protocol.py @@ -57,3 +57,17 @@ class ArtifactBackend(Protocol): artifact_id: The identifier of the artifact to remove. """ ... + + +class ArtifactStoreWrapper: + def __init__(self, artifact_backend: ArtifactBackend) -> None: + self._backend = artifact_backend + + def open_reader(self, artifact_id: str) -> BinaryIO: + return self._backend.open(artifact_id) + + def write(self, artifact_id: str, content_body: BinaryIO) -> None: + self._backend.write(artifact_id, content_body) + + def remove(self, artifact_id: str) -> None: + self._backend.remove(artifact_id)