mirror of
https://github.com/wassname/optuna-dashboard.git
synced 2026-09-17 12:30:35 +08:00
Merge pull request #684 from toshihikoyanase/replace-example-files-with-links
Replace example files with links.
This commit is contained in:
Naoto Mizuno
@@ -39,5 +39,4 @@ coverage.xml
|
||||
.vscode/
|
||||
.DS_Store
|
||||
tmp/
|
||||
examples/preferential-optimization/artifact/
|
||||
|
||||
|
||||
+14
-14
@@ -95,7 +95,7 @@ Given the above system, we carry out HITL optimization as follows:
|
||||
Environment setup
|
||||
^^^^^^^^^^^^^^^^^
|
||||
|
||||
To run `the script <https://github.com/optuna/optuna-dashboard/blob/main/examples/hitl/main.py>`_ used in this tutorial, you need to install following libraries:
|
||||
To run `the script <https://github.com/optuna/optuna-examples/blob/main/dashboard/hitl/main.py>`_ used in this tutorial, you need to install following libraries:
|
||||
|
||||
.. code-block:: console
|
||||
|
||||
@@ -109,7 +109,7 @@ You will use SQLite for the storage backend in this tutorial. Ensure that the fo
|
||||
Execution of the HITL optimization script
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
Run a python script below which you copied from `main.py <https://github.com/optuna/optuna-dashboard/blob/main/examples/hitl/main.py>`_
|
||||
Run a python script below which you copied from `main.py <https://github.com/optuna/optuna-examples/blob/main/dashboard/hitl/main.py>`_
|
||||
|
||||
.. code-block:: console
|
||||
|
||||
@@ -150,7 +150,7 @@ Click the third item in the sidebar. You will see a list of all trials.
|
||||
|
||||
.. image:: ./images/hitl10.png
|
||||
|
||||
For each trial, you can see its details such as RGB parameter values and importantly, the generated image based on these values.
|
||||
For each trial, you can see its details such as RGB parameter values and importantly, the generated image based on these values.
|
||||
|
||||
.. image:: ./images/hitl11.gif
|
||||
:width: 90%
|
||||
@@ -189,21 +189,21 @@ Let’s walk through the script we used for the optimization.
|
||||
r = trial.suggest_int("r", 0, 255)
|
||||
g = trial.suggest_int("g", 0, 255)
|
||||
b = trial.suggest_int("b", 0, 255)
|
||||
|
||||
|
||||
# 2. Generate image
|
||||
image_path = f"tmp/sample-{trial.number}.png"
|
||||
image = Image.new("RGB", (320, 240), color=(r, g, b))
|
||||
image.save(image_path)
|
||||
|
||||
|
||||
# 3. Upload Artifact
|
||||
artifact_id = upload_artifact(trial, image_path, artifact_store)
|
||||
artifact_path = get_artifact_path(trial, artifact_id)
|
||||
|
||||
|
||||
# 4. Save Note
|
||||
note = textwrap.dedent(
|
||||
f"""\
|
||||
## Trial {trial.number}
|
||||
|
||||
|
||||
|
||||
"""
|
||||
)
|
||||
@@ -222,10 +222,10 @@ In the ``suggest_and_generate_image`` function, a new Trial is obtained and new
|
||||
sampler=optuna.samplers.TPESampler(constant_liar=True, n_startup_trials=5),
|
||||
load_if_exists=True,
|
||||
)
|
||||
|
||||
|
||||
# 2. Set an objective name
|
||||
study.set_metric_names(["Looks like sunset color?"])
|
||||
|
||||
|
||||
# 3. Register ChoiceWidget
|
||||
register_objective_form_widgets(
|
||||
study,
|
||||
@@ -237,7 +237,7 @@ In the ``suggest_and_generate_image`` function, a new Trial is obtained and new
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
# 4. Start Human-in-the-loop Optimization
|
||||
n_batch = 4
|
||||
while True:
|
||||
@@ -259,17 +259,17 @@ The function ``start_optimization`` defines our loop for HITL optimization to ge
|
||||
|
||||
def main() -> NoReturn:
|
||||
tmp_path = os.path.join(os.path.dirname(__file__), "tmp")
|
||||
|
||||
|
||||
# 1. Create Artifact Store
|
||||
artifact_path = os.path.join(os.path.dirname(__file__), "artifact")
|
||||
artifact_store = FileSystemArtifactStore(artifact_path)
|
||||
|
||||
|
||||
if not os.path.exists(artifact_path):
|
||||
os.mkdir(artifact_path)
|
||||
|
||||
|
||||
if not os.path.exists(tmp_path):
|
||||
os.mkdir(tmp_path)
|
||||
|
||||
|
||||
# 2. Run optimize loop
|
||||
start_optimization(artifact_store)
|
||||
|
||||
|
||||
@@ -99,7 +99,7 @@ enabling the Optuna Dashboard to display images on the evaluation feedback page.
|
||||
register_preference_feedback_component(study, "artifact", user_attr_key)
|
||||
|
||||
Following this, we create a loop that continuously checks if new trials should be generated, awaiting human evaluation if not.
|
||||
Within the while loop, new trials are generated if the condition :meth:`~optuna_dashboard.preferential.PreferentialStudy.should_generate` returns ``True``.
|
||||
Within the while loop, new trials are generated if the condition :meth:`~optuna_dashboard.preferential.PreferentialStudy.should_generate` returns ``True``.
|
||||
For each trial, RGB values are sampled, an image is generated with these values, saved temporarily.
|
||||
Then the image is uploaded to the artifact store, and finally, the ``artifact_id`` is stored to the key, which is specified via :func:`~optuna_dashboard.register_preference_feedback_component`.
|
||||
|
||||
@@ -127,4 +127,4 @@ Then the image is uploaded to the artifact store, and finally, the ``artifact_id
|
||||
artifact_id = upload_artifact(trial, image_path, artifact_store)
|
||||
trial.set_user_attr(user_attr_key, artifact_id)
|
||||
|
||||
.. _generator.py: https://github.com/optuna/optuna-dashboard/blob/main/examples/preferential-optimization/generator.py
|
||||
.. _generator.py: https://github.com/optuna/optuna-examples/blob/main/dashboard/preferential-optimization/generator.py
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
Optuna Dashboard Examples
|
||||
=========================
|
||||
|
||||
Example files have been moved to the [optuna/optuna-examples](https://github.com/optuna/optuna-examples/) repoistory.
|
||||
You can find the dashboard-related examples in the [dashboard](https://github.com/optuna/optuna-examples/tree/main/dashboard) directory.
|
||||
@@ -1,98 +0,0 @@
|
||||
import os
|
||||
import textwrap
|
||||
import time
|
||||
from typing import NoReturn
|
||||
|
||||
import optuna
|
||||
from optuna.artifacts import FileSystemArtifactStore
|
||||
from optuna.artifacts import upload_artifact
|
||||
from optuna.trial import TrialState
|
||||
from optuna_dashboard import ChoiceWidget
|
||||
from optuna_dashboard import register_objective_form_widgets
|
||||
from optuna_dashboard import save_note
|
||||
from optuna_dashboard.artifact import get_artifact_path
|
||||
from PIL import Image
|
||||
|
||||
|
||||
def suggest_and_generate_image(
|
||||
study: optuna.Study, artifact_store: FileSystemArtifactStore
|
||||
) -> None:
|
||||
# 1. Ask new parameters
|
||||
trial = study.ask()
|
||||
r = trial.suggest_int("r", 0, 255)
|
||||
g = trial.suggest_int("g", 0, 255)
|
||||
b = trial.suggest_int("b", 0, 255)
|
||||
|
||||
# 2. Generate image
|
||||
image_path = f"tmp/sample-{trial.number}.png"
|
||||
image = Image.new("RGB", (320, 240), color=(r, g, b))
|
||||
image.save(image_path)
|
||||
|
||||
# 3. Upload Artifact
|
||||
artifact_id = upload_artifact(trial, image_path, artifact_store)
|
||||
artifact_path = get_artifact_path(trial, artifact_id)
|
||||
|
||||
# 4. Save Note
|
||||
note = textwrap.dedent(
|
||||
f"""\
|
||||
## Trial {trial.number}
|
||||
|
||||
|
||||
"""
|
||||
)
|
||||
save_note(trial, note)
|
||||
|
||||
|
||||
def start_optimization(artifact_store: FileSystemArtifactStore) -> NoReturn:
|
||||
# 1. Create Study
|
||||
study = optuna.create_study(
|
||||
study_name="Human-in-the-loop Optimization",
|
||||
storage="sqlite:///db.sqlite3",
|
||||
sampler=optuna.samplers.TPESampler(constant_liar=True, n_startup_trials=5),
|
||||
load_if_exists=True,
|
||||
)
|
||||
|
||||
# 2. Set an objective name
|
||||
study.set_metric_names(["Looks like sunset color?"])
|
||||
|
||||
# 3. Register ChoiceWidget
|
||||
register_objective_form_widgets(
|
||||
study,
|
||||
widgets=[
|
||||
ChoiceWidget(
|
||||
choices=["Good 👍", "So-so👌", "Bad 👎"],
|
||||
values=[-1, 0, 1],
|
||||
description="Please input your score!",
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
# 4. Start Human-in-the-loop Optimization
|
||||
n_batch = 4
|
||||
while True:
|
||||
running_trials = study.get_trials(deepcopy=False, states=(TrialState.RUNNING,))
|
||||
if len(running_trials) >= n_batch:
|
||||
time.sleep(1) # Avoid busy-loop
|
||||
continue
|
||||
suggest_and_generate_image(study, artifact_store)
|
||||
|
||||
|
||||
def main() -> NoReturn:
|
||||
tmp_path = os.path.join(os.path.dirname(__file__), "tmp")
|
||||
|
||||
# 1. Create Artifact Store
|
||||
artifact_path = os.path.join(os.path.dirname(__file__), "artifact")
|
||||
artifact_store = FileSystemArtifactStore(artifact_path)
|
||||
|
||||
if not os.path.exists(artifact_path):
|
||||
os.mkdir(artifact_path)
|
||||
|
||||
if not os.path.exists(tmp_path):
|
||||
os.mkdir(tmp_path)
|
||||
|
||||
# 2. Run optimize loop
|
||||
start_optimization(artifact_store)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,2 +0,0 @@
|
||||
#!/usr/bin/env sh
|
||||
optuna-dashboard sqlite:///example.db --artifact-dir ./artifact
|
||||
@@ -1,60 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import tempfile
|
||||
import time
|
||||
from typing import NoReturn
|
||||
|
||||
from optuna.artifacts import FileSystemArtifactStore
|
||||
from optuna.artifacts import upload_artifact
|
||||
from optuna_dashboard import register_preference_feedback_component
|
||||
from optuna_dashboard.preferential import create_study
|
||||
from optuna_dashboard.preferential.samplers.gp import PreferentialGPSampler
|
||||
from PIL import Image
|
||||
|
||||
|
||||
STORAGE_URL = "sqlite:///example.db"
|
||||
artifact_path = os.path.join(os.path.dirname(__file__), "artifact")
|
||||
artifact_store = FileSystemArtifactStore(base_path=artifact_path)
|
||||
os.makedirs(artifact_path, exist_ok=True)
|
||||
|
||||
|
||||
def main() -> NoReturn:
|
||||
study = create_study(
|
||||
n_generate=4,
|
||||
study_name="Preferential Optimization",
|
||||
storage=STORAGE_URL,
|
||||
sampler=PreferentialGPSampler(),
|
||||
load_if_exists=True,
|
||||
)
|
||||
# Change the component, displayed on the human feedback pages.
|
||||
# By default (component_type="note"), the Trial's Markdown note is displayed.
|
||||
user_attr_key = "rgb_image"
|
||||
register_preference_feedback_component(study, "artifact", user_attr_key)
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
while True:
|
||||
# If study.should_generate() returns False,
|
||||
# the generator waits for human evaluation.
|
||||
if not study.should_generate():
|
||||
time.sleep(0.1) # Avoid busy-loop
|
||||
continue
|
||||
|
||||
trial = study.ask()
|
||||
# 1. Ask new parameters
|
||||
r = trial.suggest_int("r", 0, 255)
|
||||
g = trial.suggest_int("g", 0, 255)
|
||||
b = trial.suggest_int("b", 0, 255)
|
||||
|
||||
# 2. Generate image
|
||||
image_path = os.path.join(tmpdir, f"sample-{trial.number}.png")
|
||||
image = Image.new("RGB", (320, 240), color=(r, g, b))
|
||||
image.save(image_path)
|
||||
|
||||
# 3. Upload Artifact and set artifact_id to trial.user_attrs["rgb_image"].
|
||||
artifact_id = upload_artifact(trial, image_path, artifact_store)
|
||||
trial.set_user_attr(user_attr_key, artifact_id)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,54 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import tempfile
|
||||
import uuid
|
||||
|
||||
import optuna
|
||||
from optuna.trial import TrialState
|
||||
from optuna_dashboard.artifact.file_system import FileSystemBackend
|
||||
from optuna_dashboard.streamlit import render_objective_form_widgets
|
||||
from optuna_dashboard.streamlit import render_trial_note
|
||||
|
||||
import streamlit as st
|
||||
|
||||
|
||||
artifact_path = os.path.join(os.path.dirname(__file__), "artifact")
|
||||
artifact_backend = FileSystemBackend(base_path=artifact_path)
|
||||
|
||||
|
||||
def get_tmp_dir() -> str:
|
||||
if "tmp_dir" not in st.session_state:
|
||||
tmp_dir_name = str(uuid.uuid4())
|
||||
tmp_dir_path = os.path.join(tempfile.gettempdir(), tmp_dir_name)
|
||||
os.makedirs(tmp_dir_path, exist_ok=True)
|
||||
st.session_state.tmp_dir = tmp_dir_path
|
||||
|
||||
return st.session_state.tmp_dir
|
||||
|
||||
|
||||
def start_streamlit() -> None:
|
||||
tmpdir = get_tmp_dir()
|
||||
study = optuna.load_study(
|
||||
storage="sqlite:///streamlit-db.sqlite3", study_name="Human-in-the-loop Optimization"
|
||||
)
|
||||
selected_trial = st.sidebar.selectbox("Trial", study.trials, format_func=lambda t: t.number)
|
||||
|
||||
if selected_trial is None:
|
||||
return
|
||||
render_trial_note(study, selected_trial)
|
||||
artifact_id = selected_trial.user_attrs.get("artifact_id")
|
||||
if artifact_id:
|
||||
with artifact_backend.open(artifact_id) as fsrc:
|
||||
tmp_img_path = os.path.join(tmpdir, artifact_id + ".png")
|
||||
with open(tmp_img_path, "wb") as fdst:
|
||||
shutil.copyfileobj(fsrc, fdst)
|
||||
st.image(tmp_img_path, caption="Image")
|
||||
|
||||
if selected_trial.state == TrialState.RUNNING:
|
||||
render_objective_form_widgets(study, selected_trial)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
start_streamlit()
|
||||
@@ -1,81 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import tempfile
|
||||
import time
|
||||
from typing import NoReturn
|
||||
|
||||
import optuna
|
||||
from optuna.trial import TrialState
|
||||
from optuna_dashboard import ChoiceWidget
|
||||
from optuna_dashboard import register_objective_form_widgets
|
||||
from optuna_dashboard import save_note
|
||||
from optuna_dashboard.artifact import upload_artifact
|
||||
from optuna_dashboard.artifact.file_system import FileSystemBackend
|
||||
from PIL import Image
|
||||
|
||||
|
||||
def suggest_and_generate_image(
|
||||
study: optuna.Study, artifact_backend: FileSystemBackend, tmpdir: str
|
||||
) -> None:
|
||||
# 1. Ask new parameters
|
||||
trial = study.ask()
|
||||
r = trial.suggest_int("r", 0, 255)
|
||||
g = trial.suggest_int("g", 0, 255)
|
||||
b = trial.suggest_int("b", 0, 255)
|
||||
|
||||
# 2. Generate image
|
||||
image_path = os.path.join(tmpdir, f"sample-{trial.number}.png")
|
||||
image = Image.new("RGB", (320, 240), color=(r, g, b))
|
||||
image.save(image_path)
|
||||
|
||||
# 3. Upload Artifact
|
||||
artifact_id = upload_artifact(artifact_backend, trial, image_path)
|
||||
trial.set_user_attr("artifact_id", artifact_id)
|
||||
|
||||
# 4. Save Note
|
||||
save_note(trial, f"## Trial {trial.number}")
|
||||
|
||||
|
||||
def main() -> NoReturn:
|
||||
# 1. Create Artifact Store
|
||||
artifact_path = os.path.join(os.path.dirname(__file__), "artifact")
|
||||
artifact_backend = FileSystemBackend(base_path=artifact_path)
|
||||
|
||||
if not os.path.exists(artifact_path):
|
||||
os.mkdir(artifact_path)
|
||||
|
||||
# 2. Create Study
|
||||
study = optuna.create_study(
|
||||
study_name="Human-in-the-loop Optimization",
|
||||
storage="sqlite:///streamlit-db.sqlite3",
|
||||
sampler=optuna.samplers.TPESampler(constant_liar=True, n_startup_trials=5),
|
||||
load_if_exists=True,
|
||||
)
|
||||
study.set_metric_names(["Looks like sunset color?"])
|
||||
|
||||
# 4. Register ChoiceWidget
|
||||
register_objective_form_widgets(
|
||||
study,
|
||||
widgets=[
|
||||
ChoiceWidget(
|
||||
choices=["Good 👍", "So-so👌", "Bad 👎"],
|
||||
values=[-1, 0, 1],
|
||||
description="Please input your score!",
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
# 5. Start Human-in-the-loop Optimization
|
||||
n_batch = 4
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
while True:
|
||||
running_trials = study.get_trials(deepcopy=False, states=(TrialState.RUNNING,))
|
||||
if len(running_trials) >= n_batch:
|
||||
time.sleep(1) # Avoid busy-loop
|
||||
continue
|
||||
suggest_and_generate_image(study, artifact_backend, tmpdir)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user