Replace ArtifactBackend with ArtifactStore

This commit is contained in:
c-bata
2023-08-04 17:36:00 +09:00
parent ccfaa0d41c
commit bdd7056394
6 changed files with 126 additions and 18 deletions
+38 -10
View File
@@ -7,6 +7,7 @@ import typing
from typing import Any
from typing import Optional
from typing import Union
import warnings
from bottle import Bottle
from bottle import redirect
@@ -36,10 +37,13 @@ from ._storage import get_trials
from ._storage_url import get_storage
from .artifact._backend import delete_all_artifacts
from .artifact._backend import register_artifact_route
from .artifact._backend_to_store import ArtifactBackendToStore
from .artifact._backend_to_store import is_artifact_store
if typing.TYPE_CHECKING:
from _typeshed.wsgi import WSGIApplication
from optuna.artifacts._protocol import ArtifactStore
from optuna_dashboard.artifact.protocol import ArtifactBackend
@@ -54,7 +58,7 @@ cached_path_exists = functools.lru_cache(maxsize=10)(os.path.exists)
def create_app(
storage: BaseStorage,
artifact_backend: Optional[ArtifactBackend] = None,
artifact_store: Optional[ArtifactStore] = None,
debug: bool = False,
) -> Bottle:
app = Bottle()
@@ -76,7 +80,7 @@ def create_app(
@json_api_view
def api_meta() -> dict[str, Any]:
return {
"artifact_is_available": artifact_backend is not None,
"artifact_is_available": artifact_store is not None,
}
@app.get("/api/studies")
@@ -156,9 +160,9 @@ def create_app(
@app.delete("/api/studies/<study_id:int>")
@json_api_view
def delete_study(study_id: int) -> dict[str, Any]:
if artifact_backend is not None:
if artifact_store is not None:
system_attrs = storage.get_study_system_attrs(study_id)
delete_all_artifacts(artifact_backend, system_attrs)
delete_all_artifacts(artifact_store, system_attrs)
try:
storage.delete_study(study_id)
@@ -347,7 +351,7 @@ def create_app(
return static_file(filename, root=STATIC_DIR)
register_rdb_migration_route(app, storage)
register_artifact_route(app, storage, artifact_backend)
register_artifact_route(app, storage, artifact_store)
return app
@@ -355,25 +359,49 @@ def run_server(
storage: Union[str, BaseStorage],
host: str = "localhost",
port: int = 8080,
artifact_backend: Optional[ArtifactBackend] = None,
artifact_store: Optional[ArtifactStore | ArtifactBackend] = None,
*,
# TODO(c-bata): Remove this keyword argument in the v0.14.0 release.
artifact_backend: Optional[ArtifactBackend | ArtifactStore] = None,
) -> None:
"""Start running optuna-dashboard and blocks until the server terminates.
This function uses wsgiref module which is not intended for the production
use. If you want to run optuna-dashboard more secure and/or more fast,
please use WSGI server like Gunicorn or uWSGI via :func:`wsgi` function.
"""
app = create_app(get_storage(storage), artifact_backend=artifact_backend)
if artifact_backend is not None:
warnings.warn(
"The `artifact_backend` argument is deprecated. "
"Please use `artifact_store` instead.",
DeprecationWarning,
)
artifact_store = ArtifactBackendToStore(artifact_backend)
if not is_artifact_store(artifact_store):
artifact_store = ArtifactBackendToStore(artifact_store)
app = create_app(get_storage(storage), artifact_store=artifact_store)
run(app, host=host, port=port)
def wsgi(
storage: Union[str, BaseStorage],
artifact_store: Optional[ArtifactBackend | ArtifactStore] = None,
*,
# TODO(c-bata): Remove this keyword argument in the v0.14.0 release.
artifact_backend: Optional[ArtifactBackend] = None,
) -> WSGIApplication:
"""This function exposes WSGI interface for people who want to run on the
production-class WSGI servers like Gunicorn or uWSGI.
"""
return create_app(get_storage(storage), artifact_backend=artifact_backend)
if artifact_backend is not None:
warnings.warn(
"The `artifact_backend` argument is deprecated. "
"Please use `artifact_store` instead.",
DeprecationWarning,
)
artifact_store = ArtifactBackendToStore(artifact_backend)
if not is_artifact_store(artifact_store):
artifact_store = ArtifactBackendToStore(artifact_store)
return create_app(get_storage(storage), artifact_store=artifact_store)
+21 -8
View File
@@ -6,6 +6,7 @@ import mimetypes
import os.path
from typing import TYPE_CHECKING
import uuid
import warnings
from bottle import BaseRequest
from bottle import Bottle
@@ -24,6 +25,7 @@ if TYPE_CHECKING:
from typing import Optional
from typing import TypedDict
from optuna.artifacts._protocol import ArtifactStore
from optuna.storages import BaseStorage
from .protocol import ArtifactBackend
@@ -57,11 +59,11 @@ def get_artifact_path(
def register_artifact_route(
app: Bottle, storage: BaseStorage, artifact_backend: Optional[ArtifactBackend]
app: Bottle, storage: BaseStorage, artifact_store: Optional[ArtifactStore]
) -> None:
@app.get("/artifacts/<study_id:int>/<trial_id:int>/<artifact_id:re:[0-9a-fA-F-]+>")
def proxy_artifact(study_id: int, trial_id: int, artifact_id: str) -> HTTPResponse | bytes:
if artifact_backend is None:
if artifact_store is None:
response.status = 400 # Bad Request
return b"Cannot access to the artifacts."
artifact_dict = _get_artifact_meta(storage, study_id, trial_id, artifact_id)
@@ -73,13 +75,13 @@ def register_artifact_route(
if encoding:
headers["Content-Encodings"] = encoding
fp = artifact_backend.open(artifact_id)
fp = artifact_store.open_reader(artifact_id)
return HTTPResponse(fp, headers=headers)
@app.post("/api/artifacts/<study_id:int>/<trial_id:int>")
@json_api_view
def upload_artifact_api(study_id: int, trial_id: int) -> dict[str, Any]:
if artifact_backend is None:
if artifact_store is None:
response.status = 400 # Bad Request
return {"reason": "Cannot access to the artifacts."}
file = request.json.get("file")
@@ -90,7 +92,7 @@ def register_artifact_route(
_, data = parse_data_uri(file)
filename = request.json.get("filename", "")
artifact_id = str(uuid.uuid4())
artifact_backend.write(artifact_id, io.BytesIO(data))
artifact_store.write(artifact_id, io.BytesIO(data))
mimetype, encoding = mimetypes.guess_type(filename)
artifact = {
@@ -115,10 +117,10 @@ def register_artifact_route(
@app.delete("/api/artifacts/<study_id:int>/<trial_id:int>/<artifact_id:re:[0-9a-fA-F-]+>")
@json_api_view
def delete_artifact(study_id: int, trial_id: int, artifact_id: str) -> dict[str, Any]:
if artifact_backend is None:
if artifact_store is None:
response.status = 400 # Bad Request
return {"reason": "Cannot access to the artifacts."}
artifact_backend.remove(artifact_id)
artifact_store.remove(artifact_id)
attr_key = _artifact_prefix(trial_id) + artifact_id
storage.set_study_system_attr(study_id, attr_key, json.dumps(None))
@@ -134,7 +136,12 @@ def upload_artifact(
mimetype: Optional[str] = None,
encoding: Optional[str] = None,
) -> str:
"""Upload an artifact (files), which is associated with the trial.
"""[Deprecated] Upload an artifact (files), which is associated with the trial.
.. note::
This function is deprecated. Please use `optuna.artifacts.upload_artifact
<https://optuna.readthedocs.io/en/latest/reference/generated/optuna.artifacts.upload_artifact.html>`_ instead.
Example:
.. code-block:: python
@@ -151,6 +158,12 @@ def upload_artifact(
upload_artifact(artifact_backend, trial, file_path)
return ...
"""
warnings.warn(
"This function is deprecated. Please use optuna.artifacts.upload_artifact() instead.\n"
"https://optuna.readthedocs.io/en/latest/reference/generated/optuna.artifacts.upload_artifact.html",
DeprecationWarning,
)
filename = os.path.basename(file_path)
storage = trial.storage
trial_id = trial._trial_id
@@ -0,0 +1,32 @@
from __future__ import annotations
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from typing import BinaryIO
from typing import TypeGuard
from optuna.artifacts._protocol import ArtifactStore
from .protocol import ArtifactBackend
def is_artifact_store(store: ArtifactBackend | ArtifactStore) -> TypeGuard[ArtifactStore]:
return getattr(store, "open_reader") is not None
class ArtifactBackendToStore:
"""Converts a Dashboard's ArtifactBackend to Optuna's ArtifactStore."""
def __init__(self, artifact_backend: ArtifactBackend) -> None:
self._backend = artifact_backend
def open_reader(self, artifact_id: str) -> BinaryIO:
return self._backend.open(artifact_id)
def write(self, artifact_id: str, content_body: BinaryIO) -> None:
self._backend.write(artifact_id, content_body)
def remove(self, artifact_id: str) -> None:
self._backend.remove(artifact_id)
+10
View File
@@ -3,6 +3,7 @@ from __future__ import annotations
import io
import shutil
from typing import TYPE_CHECKING
import warnings
import boto3
from botocore.exceptions import ClientError
@@ -19,6 +20,10 @@ if TYPE_CHECKING:
class Boto3Backend:
"""An artifact backend for S3.
.. note::
This class is deprecated. Please use `optuna.artifacts.Boto3ArtifactStore <https://optuna.readthedocs.io/en/latest/reference/generated/optuna.artifacts.Boto3ArtifactStore.html>`_ instead.",
Example:
.. code-block:: python
@@ -44,6 +49,11 @@ class Boto3Backend:
# may close the source file object.
# See https://github.com/boto/boto3/issues/929
self._avoid_buf_copy = avoid_buf_copy
warnings.warn(
"Boto3Backend is deprecated. Please use optuna.artifacts.Boto3ArtifactStore instead.\n"
"https://optuna.readthedocs.io/en/latest/reference/generated/optuna.artifacts.Boto3ArtifactStore.html",
DeprecationWarning,
)
def open(self, artifact_id: str) -> BinaryIO:
try:
+11
View File
@@ -3,6 +3,7 @@ from __future__ import annotations
import os
import shutil
from typing import TYPE_CHECKING
import warnings
from optuna_dashboard.artifact.exceptions import ArtifactNotFound
@@ -14,6 +15,11 @@ if TYPE_CHECKING:
class FileSystemBackend:
"""An artifact backend for file systems.
.. note::
This class is deprecated. Please use `optuna.artifacts.FileSystemArtifactStore
<https://optuna.readthedocs.io/en/latest/reference/generated/optuna.artifacts.FileSystemArtifactStore.html>`_ instead.",
Example:
.. code-block:: python
@@ -32,6 +38,11 @@ class FileSystemBackend:
def __init__(self, base_path: str) -> None:
self._base_path = base_path
warnings.warn(
"FileSystemBackend is deprecated. Please use optuna.artifacts.FileSystemArtifactStore instead.\n"
"https://optuna.readthedocs.io/en/latest/reference/generated/optuna.artifacts.FileSystemArtifactStore.html",
DeprecationWarning,
)
def open(self, artifact_id: str) -> BinaryIO:
filepath = os.path.join(self._base_path, artifact_id)
+14
View File
@@ -57,3 +57,17 @@ class ArtifactBackend(Protocol):
artifact_id: The identifier of the artifact to remove.
"""
...
class ArtifactStoreWrapper:
def __init__(self, artifact_backend: ArtifactBackend) -> None:
self._backend = artifact_backend
def open_reader(self, artifact_id: str) -> BinaryIO:
return self._backend.open(artifact_id)
def write(self, artifact_id: str, content_body: BinaryIO) -> None:
self._backend.write(artifact_id, content_body)
def remove(self, artifact_id: str) -> None:
self._backend.remove(artifact_id)