mirror of
https://github.com/wassname/optuna-dashboard.git
synced 2026-09-12 12:40:33 +08:00
Replace ArtifactBackend with ArtifactStore
This commit is contained in:
+38
-10
@@ -7,6 +7,7 @@ import typing
|
||||
from typing import Any
|
||||
from typing import Optional
|
||||
from typing import Union
|
||||
import warnings
|
||||
|
||||
from bottle import Bottle
|
||||
from bottle import redirect
|
||||
@@ -36,10 +37,13 @@ from ._storage import get_trials
|
||||
from ._storage_url import get_storage
|
||||
from .artifact._backend import delete_all_artifacts
|
||||
from .artifact._backend import register_artifact_route
|
||||
from .artifact._backend_to_store import ArtifactBackendToStore
|
||||
from .artifact._backend_to_store import is_artifact_store
|
||||
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from _typeshed.wsgi import WSGIApplication
|
||||
from optuna.artifacts._protocol import ArtifactStore
|
||||
from optuna_dashboard.artifact.protocol import ArtifactBackend
|
||||
|
||||
|
||||
@@ -54,7 +58,7 @@ cached_path_exists = functools.lru_cache(maxsize=10)(os.path.exists)
|
||||
|
||||
def create_app(
|
||||
storage: BaseStorage,
|
||||
artifact_backend: Optional[ArtifactBackend] = None,
|
||||
artifact_store: Optional[ArtifactStore] = None,
|
||||
debug: bool = False,
|
||||
) -> Bottle:
|
||||
app = Bottle()
|
||||
@@ -76,7 +80,7 @@ def create_app(
|
||||
@json_api_view
|
||||
def api_meta() -> dict[str, Any]:
|
||||
return {
|
||||
"artifact_is_available": artifact_backend is not None,
|
||||
"artifact_is_available": artifact_store is not None,
|
||||
}
|
||||
|
||||
@app.get("/api/studies")
|
||||
@@ -156,9 +160,9 @@ def create_app(
|
||||
@app.delete("/api/studies/<study_id:int>")
|
||||
@json_api_view
|
||||
def delete_study(study_id: int) -> dict[str, Any]:
|
||||
if artifact_backend is not None:
|
||||
if artifact_store is not None:
|
||||
system_attrs = storage.get_study_system_attrs(study_id)
|
||||
delete_all_artifacts(artifact_backend, system_attrs)
|
||||
delete_all_artifacts(artifact_store, system_attrs)
|
||||
|
||||
try:
|
||||
storage.delete_study(study_id)
|
||||
@@ -347,7 +351,7 @@ def create_app(
|
||||
return static_file(filename, root=STATIC_DIR)
|
||||
|
||||
register_rdb_migration_route(app, storage)
|
||||
register_artifact_route(app, storage, artifact_backend)
|
||||
register_artifact_route(app, storage, artifact_store)
|
||||
return app
|
||||
|
||||
|
||||
@@ -355,25 +359,49 @@ def run_server(
|
||||
storage: Union[str, BaseStorage],
|
||||
host: str = "localhost",
|
||||
port: int = 8080,
|
||||
artifact_backend: Optional[ArtifactBackend] = None,
|
||||
artifact_store: Optional[ArtifactStore | ArtifactBackend] = None,
|
||||
*,
|
||||
# TODO(c-bata): Remove this keyword argument in the v0.14.0 release.
|
||||
artifact_backend: Optional[ArtifactBackend | ArtifactStore] = None,
|
||||
) -> None:
|
||||
"""Start running optuna-dashboard and blocks until the server terminates.
|
||||
|
||||
This function uses wsgiref module which is not intended for the production
|
||||
use. If you want to run optuna-dashboard more secure and/or more fast,
|
||||
please use WSGI server like Gunicorn or uWSGI via :func:`wsgi` function.
|
||||
|
||||
|
||||
"""
|
||||
app = create_app(get_storage(storage), artifact_backend=artifact_backend)
|
||||
if artifact_backend is not None:
|
||||
warnings.warn(
|
||||
"The `artifact_backend` argument is deprecated. "
|
||||
"Please use `artifact_store` instead.",
|
||||
DeprecationWarning,
|
||||
)
|
||||
artifact_store = ArtifactBackendToStore(artifact_backend)
|
||||
if not is_artifact_store(artifact_store):
|
||||
artifact_store = ArtifactBackendToStore(artifact_store)
|
||||
|
||||
app = create_app(get_storage(storage), artifact_store=artifact_store)
|
||||
run(app, host=host, port=port)
|
||||
|
||||
|
||||
def wsgi(
|
||||
storage: Union[str, BaseStorage],
|
||||
artifact_store: Optional[ArtifactBackend | ArtifactStore] = None,
|
||||
*,
|
||||
# TODO(c-bata): Remove this keyword argument in the v0.14.0 release.
|
||||
artifact_backend: Optional[ArtifactBackend] = None,
|
||||
) -> WSGIApplication:
|
||||
"""This function exposes WSGI interface for people who want to run on the
|
||||
production-class WSGI servers like Gunicorn or uWSGI.
|
||||
"""
|
||||
return create_app(get_storage(storage), artifact_backend=artifact_backend)
|
||||
if artifact_backend is not None:
|
||||
warnings.warn(
|
||||
"The `artifact_backend` argument is deprecated. "
|
||||
"Please use `artifact_store` instead.",
|
||||
DeprecationWarning,
|
||||
)
|
||||
artifact_store = ArtifactBackendToStore(artifact_backend)
|
||||
if not is_artifact_store(artifact_store):
|
||||
artifact_store = ArtifactBackendToStore(artifact_store)
|
||||
|
||||
return create_app(get_storage(storage), artifact_store=artifact_store)
|
||||
|
||||
@@ -6,6 +6,7 @@ import mimetypes
|
||||
import os.path
|
||||
from typing import TYPE_CHECKING
|
||||
import uuid
|
||||
import warnings
|
||||
|
||||
from bottle import BaseRequest
|
||||
from bottle import Bottle
|
||||
@@ -24,6 +25,7 @@ if TYPE_CHECKING:
|
||||
from typing import Optional
|
||||
from typing import TypedDict
|
||||
|
||||
from optuna.artifacts._protocol import ArtifactStore
|
||||
from optuna.storages import BaseStorage
|
||||
|
||||
from .protocol import ArtifactBackend
|
||||
@@ -57,11 +59,11 @@ def get_artifact_path(
|
||||
|
||||
|
||||
def register_artifact_route(
|
||||
app: Bottle, storage: BaseStorage, artifact_backend: Optional[ArtifactBackend]
|
||||
app: Bottle, storage: BaseStorage, artifact_store: Optional[ArtifactStore]
|
||||
) -> None:
|
||||
@app.get("/artifacts/<study_id:int>/<trial_id:int>/<artifact_id:re:[0-9a-fA-F-]+>")
|
||||
def proxy_artifact(study_id: int, trial_id: int, artifact_id: str) -> HTTPResponse | bytes:
|
||||
if artifact_backend is None:
|
||||
if artifact_store is None:
|
||||
response.status = 400 # Bad Request
|
||||
return b"Cannot access to the artifacts."
|
||||
artifact_dict = _get_artifact_meta(storage, study_id, trial_id, artifact_id)
|
||||
@@ -73,13 +75,13 @@ def register_artifact_route(
|
||||
if encoding:
|
||||
headers["Content-Encodings"] = encoding
|
||||
|
||||
fp = artifact_backend.open(artifact_id)
|
||||
fp = artifact_store.open_reader(artifact_id)
|
||||
return HTTPResponse(fp, headers=headers)
|
||||
|
||||
@app.post("/api/artifacts/<study_id:int>/<trial_id:int>")
|
||||
@json_api_view
|
||||
def upload_artifact_api(study_id: int, trial_id: int) -> dict[str, Any]:
|
||||
if artifact_backend is None:
|
||||
if artifact_store is None:
|
||||
response.status = 400 # Bad Request
|
||||
return {"reason": "Cannot access to the artifacts."}
|
||||
file = request.json.get("file")
|
||||
@@ -90,7 +92,7 @@ def register_artifact_route(
|
||||
_, data = parse_data_uri(file)
|
||||
filename = request.json.get("filename", "")
|
||||
artifact_id = str(uuid.uuid4())
|
||||
artifact_backend.write(artifact_id, io.BytesIO(data))
|
||||
artifact_store.write(artifact_id, io.BytesIO(data))
|
||||
|
||||
mimetype, encoding = mimetypes.guess_type(filename)
|
||||
artifact = {
|
||||
@@ -115,10 +117,10 @@ def register_artifact_route(
|
||||
@app.delete("/api/artifacts/<study_id:int>/<trial_id:int>/<artifact_id:re:[0-9a-fA-F-]+>")
|
||||
@json_api_view
|
||||
def delete_artifact(study_id: int, trial_id: int, artifact_id: str) -> dict[str, Any]:
|
||||
if artifact_backend is None:
|
||||
if artifact_store is None:
|
||||
response.status = 400 # Bad Request
|
||||
return {"reason": "Cannot access to the artifacts."}
|
||||
artifact_backend.remove(artifact_id)
|
||||
artifact_store.remove(artifact_id)
|
||||
|
||||
attr_key = _artifact_prefix(trial_id) + artifact_id
|
||||
storage.set_study_system_attr(study_id, attr_key, json.dumps(None))
|
||||
@@ -134,7 +136,12 @@ def upload_artifact(
|
||||
mimetype: Optional[str] = None,
|
||||
encoding: Optional[str] = None,
|
||||
) -> str:
|
||||
"""Upload an artifact (files), which is associated with the trial.
|
||||
"""[Deprecated] Upload an artifact (files), which is associated with the trial.
|
||||
|
||||
.. note::
|
||||
|
||||
This function is deprecated. Please use `optuna.artifacts.upload_artifact
|
||||
<https://optuna.readthedocs.io/en/latest/reference/generated/optuna.artifacts.upload_artifact.html>`_ instead.
|
||||
|
||||
Example:
|
||||
.. code-block:: python
|
||||
@@ -151,6 +158,12 @@ def upload_artifact(
|
||||
upload_artifact(artifact_backend, trial, file_path)
|
||||
return ...
|
||||
"""
|
||||
warnings.warn(
|
||||
"This function is deprecated. Please use optuna.artifacts.upload_artifact() instead.\n"
|
||||
"https://optuna.readthedocs.io/en/latest/reference/generated/optuna.artifacts.upload_artifact.html",
|
||||
DeprecationWarning,
|
||||
)
|
||||
|
||||
filename = os.path.basename(file_path)
|
||||
storage = trial.storage
|
||||
trial_id = trial._trial_id
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from typing import BinaryIO
|
||||
from typing import TypeGuard
|
||||
|
||||
from optuna.artifacts._protocol import ArtifactStore
|
||||
|
||||
from .protocol import ArtifactBackend
|
||||
|
||||
|
||||
def is_artifact_store(store: ArtifactBackend | ArtifactStore) -> TypeGuard[ArtifactStore]:
|
||||
return getattr(store, "open_reader") is not None
|
||||
|
||||
|
||||
class ArtifactBackendToStore:
|
||||
"""Converts a Dashboard's ArtifactBackend to Optuna's ArtifactStore."""
|
||||
|
||||
def __init__(self, artifact_backend: ArtifactBackend) -> None:
|
||||
self._backend = artifact_backend
|
||||
|
||||
def open_reader(self, artifact_id: str) -> BinaryIO:
|
||||
return self._backend.open(artifact_id)
|
||||
|
||||
def write(self, artifact_id: str, content_body: BinaryIO) -> None:
|
||||
self._backend.write(artifact_id, content_body)
|
||||
|
||||
def remove(self, artifact_id: str) -> None:
|
||||
self._backend.remove(artifact_id)
|
||||
@@ -3,6 +3,7 @@ from __future__ import annotations
|
||||
import io
|
||||
import shutil
|
||||
from typing import TYPE_CHECKING
|
||||
import warnings
|
||||
|
||||
import boto3
|
||||
from botocore.exceptions import ClientError
|
||||
@@ -19,6 +20,10 @@ if TYPE_CHECKING:
|
||||
class Boto3Backend:
|
||||
"""An artifact backend for S3.
|
||||
|
||||
.. note::
|
||||
|
||||
This class is deprecated. Please use `optuna.artifacts.Boto3ArtifactStore <https://optuna.readthedocs.io/en/latest/reference/generated/optuna.artifacts.Boto3ArtifactStore.html>`_ instead.",
|
||||
|
||||
Example:
|
||||
.. code-block:: python
|
||||
|
||||
@@ -44,6 +49,11 @@ class Boto3Backend:
|
||||
# may close the source file object.
|
||||
# See https://github.com/boto/boto3/issues/929
|
||||
self._avoid_buf_copy = avoid_buf_copy
|
||||
warnings.warn(
|
||||
"Boto3Backend is deprecated. Please use optuna.artifacts.Boto3ArtifactStore instead.\n"
|
||||
"https://optuna.readthedocs.io/en/latest/reference/generated/optuna.artifacts.Boto3ArtifactStore.html",
|
||||
DeprecationWarning,
|
||||
)
|
||||
|
||||
def open(self, artifact_id: str) -> BinaryIO:
|
||||
try:
|
||||
|
||||
@@ -3,6 +3,7 @@ from __future__ import annotations
|
||||
import os
|
||||
import shutil
|
||||
from typing import TYPE_CHECKING
|
||||
import warnings
|
||||
|
||||
from optuna_dashboard.artifact.exceptions import ArtifactNotFound
|
||||
|
||||
@@ -14,6 +15,11 @@ if TYPE_CHECKING:
|
||||
class FileSystemBackend:
|
||||
"""An artifact backend for file systems.
|
||||
|
||||
.. note::
|
||||
|
||||
This class is deprecated. Please use `optuna.artifacts.FileSystemArtifactStore
|
||||
<https://optuna.readthedocs.io/en/latest/reference/generated/optuna.artifacts.FileSystemArtifactStore.html>`_ instead.",
|
||||
|
||||
Example:
|
||||
.. code-block:: python
|
||||
|
||||
@@ -32,6 +38,11 @@ class FileSystemBackend:
|
||||
|
||||
def __init__(self, base_path: str) -> None:
|
||||
self._base_path = base_path
|
||||
warnings.warn(
|
||||
"FileSystemBackend is deprecated. Please use optuna.artifacts.FileSystemArtifactStore instead.\n"
|
||||
"https://optuna.readthedocs.io/en/latest/reference/generated/optuna.artifacts.FileSystemArtifactStore.html",
|
||||
DeprecationWarning,
|
||||
)
|
||||
|
||||
def open(self, artifact_id: str) -> BinaryIO:
|
||||
filepath = os.path.join(self._base_path, artifact_id)
|
||||
|
||||
@@ -57,3 +57,17 @@ class ArtifactBackend(Protocol):
|
||||
artifact_id: The identifier of the artifact to remove.
|
||||
"""
|
||||
...
|
||||
|
||||
|
||||
class ArtifactStoreWrapper:
|
||||
def __init__(self, artifact_backend: ArtifactBackend) -> None:
|
||||
self._backend = artifact_backend
|
||||
|
||||
def open_reader(self, artifact_id: str) -> BinaryIO:
|
||||
return self._backend.open(artifact_id)
|
||||
|
||||
def write(self, artifact_id: str, content_body: BinaryIO) -> None:
|
||||
self._backend.write(artifact_id, content_body)
|
||||
|
||||
def remove(self, artifact_id: str) -> None:
|
||||
self._backend.remove(artifact_id)
|
||||
|
||||
Reference in New Issue
Block a user