diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 00000000..c2c7a342 --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1 @@ +github: optuna diff --git a/docs/tutorials/hitl.rst b/docs/tutorials/hitl.rst index 6d31ae55..e06709c8 100644 --- a/docs/tutorials/hitl.rst +++ b/docs/tutorials/hitl.rst @@ -1,11 +1,13 @@ -Tutorial: Human-in-the-loop Optimization -======================================== +.. _tutorial-hitl-objective-form-widgets: + +Tutorial: Human-in-the-loop Optimization using Objective Form Widgets +===================================================================== .. image:: ./images/hitl1.png In tasks involving image generation, natural language, or speech synthesis, evaluating results mechanically can be tough, and human evaluation becomes crucial. Until now, managing such tasks with Optuna has been challenging. However, the introduction of Optuna Dashboard enables humans and optimization algorithms to work interactively and execute the optimization process. -In this tutorial, we will explain how to optimize hyperparameters to generate a simple image using Optuna Dashboard. While the tutorial focuses on a simple task, the same approach can be applied to for instance optimize more complex images, natural language, and speech. +In this tutorial, we will explain how to optimize hyperparameters to generate a simple image using Optuna Dashboard. While the tutorial focuses on a simple task, the same approach can be applied to for instance optimize more complex images, natural language, and speech. The tutorial is organized as follows: @@ -93,11 +95,11 @@ Given the above system, we carry out HITL optimization as follows: Environment setup ^^^^^^^^^^^^^^^^^ -To run `the script `_ used in this tutorial, you need to install two libraries: +To run `the script `_ used in this tutorial, you need to install following libraries: .. code-block:: console - $ pip install "optuna>=3.2.0" "optuna-dashboard>=0.10.0" pillow + $ pip install "optuna>=3.3.0" "optuna-dashboard>=0.12.0" pillow You will use SQLite for the storage backend in this tutorial. Ensure that the following library is installed: @@ -179,7 +181,9 @@ Let’s walk through the script we used for the optimization. .. code-block:: python :linenos: - def suggest_and_generate_image(study: optuna.Study, artifact_backend: FileSystemBackend) -> None: + def suggest_and_generate_image( + study: optuna.Study, artifact_store: FileSystemArtifactStore + ) -> None: # 1. Ask new parameters trial = study.ask() r = trial.suggest_int("r", 0, 255) @@ -192,7 +196,7 @@ Let’s walk through the script we used for the optimization. image.save(image_path) # 3. Upload Artifact - artifact_id = upload_artifact(artifact_backend, trial, image_path) + artifact_id = upload_artifact(trial, image_path, artifact_store) artifact_path = get_artifact_path(trial, artifact_id) # 4. Save Note @@ -205,12 +209,12 @@ Let’s walk through the script we used for the optimization. ) save_note(trial, note) -In the ``suggest_and_generate_image`` function, a new Trial is obtained and new hyperparameters are suggested for that Trial. Based on those hyperparameters, an RGB image is generated as an artifact. The generated image is then uploaded to the Artifact Storage of the Optuna Dashboard, and the image is also displayed in the Dashboard's Note. For more information on how to use the Note feature, please refer to the API Reference of :func:`~optuna_dashboard.save_note`. +In the ``suggest_and_generate_image`` function, a new Trial is obtained and new hyperparameters are suggested for that Trial. Based on those hyperparameters, an RGB image is generated as an artifact. The generated image is then uploaded to the Artifact Store of the Optuna, and the image is also displayed in the Dashboard's Note. For more information on how to use the Note feature, please refer to the API Reference of :func:`~optuna_dashboard.save_note`. .. code-block:: python :linenos: - def start_optimization(artifact_backend: FileSystemBackend) -> NoReturn: + def start_optimization(artifact_store: FileSystemArtifactStore) -> NoReturn: # 1. Create Study study = optuna.create_study( study_name="Human-in-the-loop Optimization", @@ -218,10 +222,10 @@ In the ``suggest_and_generate_image`` function, a new Trial is obtained and new sampler=optuna.samplers.TPESampler(constant_liar=True, n_startup_trials=5), load_if_exists=True, ) - + # 2. Set an objective name study.set_metric_names(["Looks like sunset color?"]) - + # 3. Register ChoiceWidget register_objective_form_widgets( study, @@ -234,15 +238,14 @@ In the ``suggest_and_generate_image`` function, a new Trial is obtained and new ], ) - # 4. Start Optimization + # 4. Start Human-in-the-loop Optimization n_batch = 4 while True: running_trials = study.get_trials(deepcopy=False, states=(TrialState.RUNNING,)) if len(running_trials) >= n_batch: time.sleep(1) # Avoid busy-loop continue - suggest_and_generate_image(study, artifact_backend) - + suggest_and_generate_image(study, artifact_store) The function ``start_optimization`` defines our loop for HITL optimization to generate an image resembling a sunset color. @@ -256,10 +259,10 @@ The function ``start_optimization`` defines our loop for HITL optimization to ge def main() -> NoReturn: tmp_path = os.path.join(os.path.dirname(__file__), "tmp") - + # 1. Create Artifact Store artifact_path = os.path.join(os.path.dirname(__file__), "artifact") - artifact_backend = FileSystemBackend(base_path=artifact_path) + artifact_store = FileSystemArtifactStore(artifact_path) if not os.path.exists(artifact_path): os.mkdir(artifact_path) @@ -268,11 +271,11 @@ The function ``start_optimization`` defines our loop for HITL optimization to ge os.mkdir(tmp_path) # 2. Run optimize loop - start_optimization(artifact_backend) + start_optimization(artifact_store) In the ``main`` function, at first, the locations of the Artifact Store is set. -* At #1, the :class:`~optuna_dashboard.FileSystemBackend` is created, which is one of the Artifact Storage options used in the Optuna Dashboard. Artifact Storage is used to store artifacts (data, files, etc.) generated during Optuna trials. For more information, please refer to the API Reference. +* At #1, the `FileSystemArtifactStore `_ is created, which is one of the Artifact Store options used in the Optuna. Artifact Store is used to store artifacts (data, files, etc.) generated during Optuna trials. For more information, please refer to the API Reference. * At #2, `start_optimization()` function, which is described above, is called. After that, two folders are created, artifact and tmp, and then ``start_optimization`` function is called to start the HITL optimization using Optuna. diff --git a/docs/tutorials/images/preferential-optimization/anim.gif b/docs/tutorials/images/preferential-optimization/anim.gif new file mode 100644 index 00000000..ba15e2b5 Binary files /dev/null and b/docs/tutorials/images/preferential-optimization/anim.gif differ diff --git a/docs/tutorials/images/preferential-optimization/system-architecture.png b/docs/tutorials/images/preferential-optimization/system-architecture.png new file mode 100644 index 00000000..73dfd8d6 Binary files /dev/null and b/docs/tutorials/images/preferential-optimization/system-architecture.png differ diff --git a/docs/tutorials/index.rst b/docs/tutorials/index.rst index 5af1ba40..f396e740 100644 --- a/docs/tutorials/index.rst +++ b/docs/tutorials/index.rst @@ -5,3 +5,4 @@ Tutorials :maxdepth: 1 hitl + preferential-optimization diff --git a/docs/tutorials/preferential-optimization.rst b/docs/tutorials/preferential-optimization.rst new file mode 100644 index 00000000..706eabe8 --- /dev/null +++ b/docs/tutorials/preferential-optimization.rst @@ -0,0 +1,135 @@ +Tutorial: Preferential Optimization +=================================== + +What is Preferential Optimization? +---------------------------------- + +Preferential optimization is the way to optimize hyperparameters based on human preferences, +specifically by determining which trial is better when given a pair to compare. +Compared to the `human-in-the-loop optimization utilizing objective form widgets `_, +which relies on absolute evaluations, preferential optimization significantly reduces fluctuations in the evaluators' criteria, +ensuring more consistent results. + +In this tutorial, we will interactively optimize RGB values between 0 and 255 to generate a color that resembles the "sunset hue", which is the same problem setting as `this tutorial `_. +Hence, familiarizing yourself with the tutorial on objective form widgets beforehand might offer a smoother understanding. + +How to Run Preferential Optimization +------------------------------------ + +In preferential optimization, we run two programs simultaneously: `generator.py`_ which executes parameter sampling or image generation, +and the Optuna Dashboard which provides a user interface for human evaluation. + +.. figure:: ./images/preferential-optimization/system-architecture.png + :alt: System Architecture + :align: center + :width: 800px + +To start, ensure you have the necessary packages installed. You can do this by running the following command in your terminal: + +.. code-block:: console + + $ pip install "optuna>=3.3.0" "optuna-dashboard>=0.13.0b1" pillow botorch + +Run a Python script below which you copied from `generator.py`_. + +.. code-block:: console + + $ python generator.py + +Then run a following command to launch Optuna Dashboard in a separate process. + +.. code-block:: console + + $ optuna-dashboard sqlite:///example.db --artifact-dir ./artifact + +In the command, the storage is set to ``sqlite:///example.db`` to persist Optuna's trial history. +To store the artifacts (output images), ``--artifact-dir ./artifact`` is specified. + +.. code-block:: console + + Listening on http://127.0.0.1:8080/ + Hit Ctrl-C to quit. + +When you run the command, you will see a message like the one above. +Please open `http://127.0.0.1:8080/dashboard/ `_ in your browser, then you can see the Optuna Dashboard as follows: + +.. figure:: ./images/preferential-optimization/anim.gif + :alt: GIF animation for preferential optimization + :align: center + :width: 800px + + Selecting the least sunset-like color from four trials to report human preferences. + + +Script Explanation +------------------ + +Here, we specify the SQLite database URL and setup the artifact store, a filesystem to store images generated during the trial. + +.. code-block:: python + :linenos: + + STORAGE_URL = "sqlite:///example.db" + artifact_path = os.path.join(os.path.dirname(__file__), "artifact") + artifact_store = FileSystemArtifactStore(base_path=artifact_path) + os.makedirs(artifact_path, exist_ok=True) + +Within the ``main()`` function, we initialize the study with necessary parameters, including specifying the preferential sampler. +ote that the ``Study`` and ``Sampler`` instantiated here are different from the conventional Optuna's ``Study``` and the ``Sampler``. +Preferential optimization relies solely on the comparison results between trials, and there are no absolute evaluation values for each trial. +Therefore, it is necessary to create dedicated ``Study`` and ``Sampler`` objects. + +.. code-block:: python + :linenos: + + from optuna_dashboard.preferential import create_study + from optuna_dashboard.preferential.samplers.gp import PreferentialGPSampler + + study = create_study( + n_generate=5, + study_name="Preferential Optimization", + storage=STORAGE_URL, + sampler=PreferentialGPSampler(), + load_if_exists=True, + ) + +Then, we create a loop that continuously checks if new trials should be generated, awaiting human evaluation if not. +Within the while loop, new trials are generated if the condition :meth:`~optuna_dashboard.preferential.PreferentialStudy.should_generate` returns ``True``. +For each trial, RGB values are sampled, and an image is generated with these values. +The image is saved temporarily, uploaded to artifact store, and then saved a Markdown note using :func:`~optuna_dashboard.save_note`. + +.. code-block:: python + :linenos: + + while True: + # If study.should_generate() returns False, the generator waits for human evaluation. + if not study.should_generate(): + time.sleep(0.1) # Avoid busy-loop + continue + + trial = study.ask() + # Ask new parameters + r = trial.suggest_int("r", 0, 255) + g = trial.suggest_int("g", 0, 255) + b = trial.suggest_int("b", 0, 255) + + # Generate an image + image_path = os.path.join(tmpdir, f"sample-{trial.number}.png") + image = Image.new("RGB", (320, 240), color=(r, g, b)) + image.save(image_path) + + # Upload to Artifact store + artifact_id = upload_artifact(trial, image_path, artifact_store) + trial.set_user_attr("artifact_id", artifact_id) + print("RGB:", (r, g, b)) + + # Save a Markdown note + note = textwrap.dedent( + f"""\ + ![generated-image]({get_artifact_path(trial, artifact_id)}) + + (R, G, B) = ({r}, {g}, {b}) + """ + ) + +.. _generator.py: https://github.com/optuna/optuna-dashboard/blob/main/examples/preferential-optimization/generator.py diff --git a/examples/hitl/main.py b/examples/hitl/main.py index 1557ae21..7844bc7a 100644 --- a/examples/hitl/main.py +++ b/examples/hitl/main.py @@ -4,17 +4,19 @@ import time from typing import NoReturn import optuna +from optuna.artifacts import FileSystemArtifactStore +from optuna.artifacts import upload_artifact from optuna.trial import TrialState from optuna_dashboard import ChoiceWidget from optuna_dashboard import register_objective_form_widgets from optuna_dashboard import save_note from optuna_dashboard.artifact import get_artifact_path -from optuna_dashboard.artifact import upload_artifact -from optuna_dashboard.artifact.file_system import FileSystemBackend from PIL import Image -def suggest_and_generate_image(study: optuna.Study, artifact_backend: FileSystemBackend) -> None: +def suggest_and_generate_image( + study: optuna.Study, artifact_store: FileSystemArtifactStore +) -> None: # 1. Ask new parameters trial = study.ask() r = trial.suggest_int("r", 0, 255) @@ -27,7 +29,7 @@ def suggest_and_generate_image(study: optuna.Study, artifact_backend: FileSystem image.save(image_path) # 3. Upload Artifact - artifact_id = upload_artifact(artifact_backend, trial, image_path) + artifact_id = upload_artifact(trial, image_path, artifact_store) artifact_path = get_artifact_path(trial, artifact_id) # 4. Save Note @@ -41,7 +43,7 @@ def suggest_and_generate_image(study: optuna.Study, artifact_backend: FileSystem save_note(trial, note) -def start_optimization(artifact_backend: FileSystemBackend) -> NoReturn: +def start_optimization(artifact_store: FileSystemArtifactStore) -> NoReturn: # 1. Create Study study = optuna.create_study( study_name="Human-in-the-loop Optimization", @@ -72,7 +74,7 @@ def start_optimization(artifact_backend: FileSystemBackend) -> NoReturn: if len(running_trials) >= n_batch: time.sleep(1) # Avoid busy-loop continue - suggest_and_generate_image(study, artifact_backend) + suggest_and_generate_image(study, artifact_store) def main() -> NoReturn: @@ -80,7 +82,7 @@ def main() -> NoReturn: # 1. Create Artifact Store artifact_path = os.path.join(os.path.dirname(__file__), "artifact") - artifact_backend = FileSystemBackend(base_path=artifact_path) + artifact_store = FileSystemArtifactStore(artifact_path) if not os.path.exists(artifact_path): os.mkdir(artifact_path) @@ -89,7 +91,7 @@ def main() -> NoReturn: os.mkdir(tmp_path) # 2. Run optimize loop - start_optimization(artifact_backend) + start_optimization(artifact_store) if __name__ == "__main__": diff --git a/examples/preferential-optimization/generator.py b/examples/preferential-optimization/generator.py index d8f7d3bd..66be1765 100644 --- a/examples/preferential-optimization/generator.py +++ b/examples/preferential-optimization/generator.py @@ -6,10 +6,10 @@ import textwrap import time from typing import NoReturn +from optuna.artifacts import FileSystemArtifactStore +from optuna.artifacts import upload_artifact from optuna_dashboard import save_note from optuna_dashboard.artifact import get_artifact_path -from optuna_dashboard.artifact import upload_artifact -from optuna_dashboard.artifact.file_system import FileSystemBackend from optuna_dashboard.preferential import create_study from optuna_dashboard.preferential.samplers.gp import PreferentialGPSampler from PIL import Image @@ -17,7 +17,7 @@ from PIL import Image STORAGE_URL = "sqlite:///example.db" artifact_path = os.path.join(os.path.dirname(__file__), "artifact") -artifact_backend = FileSystemBackend(base_path=artifact_path) +artifact_store = FileSystemArtifactStore(base_path=artifact_path) os.makedirs(artifact_path, exist_ok=True) @@ -32,7 +32,7 @@ def main() -> NoReturn: with tempfile.TemporaryDirectory() as tmpdir: while True: - # If n_comparison "best" trials (that are not reported bad) exists, + # If study.should_generate() returns False, # the generator waits for human evaluation. if not study.should_generate(): time.sleep(0.1) # Avoid busy-loop @@ -50,7 +50,7 @@ def main() -> NoReturn: image.save(image_path) # 3. Upload Artifact - artifact_id = upload_artifact(artifact_backend, trial, image_path) + artifact_id = upload_artifact(trial, image_path, artifact_store) trial.set_user_attr("artifact_id", artifact_id) print("RGB:", (r, g, b)) diff --git a/optuna_dashboard/ts/components/PreferentialTrials.tsx b/optuna_dashboard/ts/components/PreferentialTrials.tsx index e68e7ec9..b7805259 100644 --- a/optuna_dashboard/ts/components/PreferentialTrials.tsx +++ b/optuna_dashboard/ts/components/PreferentialTrials.tsx @@ -29,7 +29,6 @@ import { } from "./ThreejsArtifactViewer" import { ArtifactCardMedia } from "./ArtifactCardMedia" import { MarkdownRenderer } from "./Note" -import { Details } from "@mui/icons-material" const ModalPage: FC<{ children: React.ReactNode