diff --git a/docs/api.rst b/docs/api.rst index 18f8bc72..e26dcfd6 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -44,6 +44,7 @@ Preferential Optimization optuna_dashboard.preferential.create_study optuna_dashboard.preferential.load_study optuna_dashboard.preferential.PreferentialStudy + optuna_dashboard.preferential.samplers.gp.PreferentialGPSampler optuna_dashboard.register_preference_feedback_component Streamlit diff --git a/docs/tutorials/preferential-optimization.rst b/docs/tutorials/preferential-optimization.rst index 4d2644ea..a192e883 100644 --- a/docs/tutorials/preferential-optimization.rst +++ b/docs/tutorials/preferential-optimization.rst @@ -27,7 +27,7 @@ First, ensure the necessary packages are installed by executing the following co .. code-block:: console - $ pip install "optuna>=3.3.0" "optuna-dashboard>=0.13.0b1" pillow botorch + $ pip install "optuna>=3.3.0" "optuna-dashboard[preferential]>=0.13.0b1" pillow Next, execute the Python script, copied from `generator.py`_. diff --git a/optuna_dashboard/preferential/_study.py b/optuna_dashboard/preferential/_study.py index d715b97b..101b61de 100644 --- a/optuna_dashboard/preferential/_study.py +++ b/optuna_dashboard/preferential/_study.py @@ -7,9 +7,9 @@ from typing import Iterable import optuna from optuna import logging +from optuna._imports import try_import from optuna.distributions import BaseDistribution from optuna.samplers import BaseSampler -from optuna.samplers import RandomSampler from optuna.trial import FrozenTrial from optuna.trial import TrialState from optuna_dashboard.preferential._system_attrs import get_n_generate @@ -20,6 +20,10 @@ from optuna_dashboard.preferential._system_attrs import report_preferences from optuna_dashboard.preferential._system_attrs import set_n_generate +with try_import() as _imports: + from optuna_dashboard.preferential.samplers.gp import PreferentialGPSampler + + _logger = logging.get_logger(__name__) _SYSTEM_ATTR_PREFERENTIAL_STUDY = "preference:is_preferential" @@ -344,11 +348,10 @@ def create_study( sampler: A sampler object that implements background algorithm for value suggestion. - If :obj:`None` is specified, `RandomSampler`_ is used. Please note that - most Optuna samplers does not work efficiently for preferential optimization. - - .. _RandomSampler: https://optuna.readthedocs.io/en/stable/reference/\ - samplers/generated/optuna.samplers.RandomSampler.html + If :obj:`None` is specified, + :class:`~optuna_dashboard.preferential.samplers.gp.PreferentialGPSampler` is used. + Please note that most Optuna samplers does not work efficiently for preferential + optimization. study_name: Study's name. If this argument is set to None, a unique name is generated @@ -369,9 +372,13 @@ def create_study( The interface may change in newer versions without prior notice. """ try: + if sampler is None: + _imports.check() # If BoTorch is not installed, raise ImportError. + sampler = PreferentialGPSampler() + study = optuna.create_study( storage=storage, - sampler=sampler or RandomSampler(), + sampler=sampler, study_name=study_name, ) study._storage.set_study_system_attr( @@ -441,11 +448,10 @@ def load_study( :func:`~optuna.study.create_study` for further details. sampler: A sampler object that implements background algorithm for value suggestion. - If :obj:`None` is specified, `RandomSampler`_ is used. Please note that - most Optuna samplers does not work efficiently for preferential optimization. - - .. _RandomSampler: https://optuna.readthedocs.io/en/stable/reference/samplers/\ - generated/optuna.samplers.RandomSampler.html + If :obj:`None` is specified, + :class:`~optuna_dashboard.preferential.samplers.gp.PreferentialGPSampler` is used. + Please note that most Optuna samplers does not work efficiently for preferential + optimization. Returns: A :class:`~optuna_dashboard.preferential.PreferentialStudy` object. @@ -454,9 +460,11 @@ def load_study( Preferential optimization is an experimental feature (introduced in v0.13.0). The interface may change in newer versions without prior notice. """ - study = optuna.load_study( - study_name=study_name, storage=storage, sampler=sampler or RandomSampler() - ) + if sampler is None: + _imports.check() # If BoTorch is not installed, raise ImportError. + sampler = PreferentialGPSampler() + + study = optuna.load_study(study_name=study_name, storage=storage, sampler=sampler) system_attrs = study._storage.get_study_system_attrs(study._study_id) if not system_attrs.get(_SYSTEM_ATTR_PREFERENTIAL_STUDY): raise ValueError("The study is not a PreferentialStudy.") diff --git a/optuna_dashboard/preferential/samplers/gp.py b/optuna_dashboard/preferential/samplers/gp.py index 457379dc..3cd98061 100644 --- a/optuna_dashboard/preferential/samplers/gp.py +++ b/optuna_dashboard/preferential/samplers/gp.py @@ -284,6 +284,26 @@ class _PreferentialGP: class PreferentialGPSampler(optuna.samplers.BaseSampler): + """Sampler for preferential optimization using Gaussian process. + + The sampling algorithm is based on `Takeno et al., 2023 `_. + This sampler uses BoTorch to optimize acquisition function. + + Args: + kernel: + Kernel that computes the covariance on the Gaussian process. Defaults to + Matern 3/2 Kernel + ARD. + noise_prior: + Prior of the observation noise. Defaults to gamma prior. + independent_sampler: + A :class:`~optuna.samplers.BaseSampler` instance that is used for independent + sampling. The parameters not contained in the relative search space are sampled + by this sampler. If :obj:`None` is specified, + :class:`~optuna.samplers.RandomSampler` is used as the default. + seed: + Seed for random number generator. + """ + def __init__( self, *, diff --git a/optuna_dashboard/ts/components/PreferenceHistory.tsx b/optuna_dashboard/ts/components/PreferenceHistory.tsx index 6aa67317..b7bd04b4 100644 --- a/optuna_dashboard/ts/components/PreferenceHistory.tsx +++ b/optuna_dashboard/ts/components/PreferenceHistory.tsx @@ -147,6 +147,16 @@ const CandidateTrial: FC<{ overflow: "auto", }} > + setDetailShown(false)} + > + + false} diff --git a/optuna_dashboard/ts/components/ThreejsArtifactViewer.tsx b/optuna_dashboard/ts/components/ThreejsArtifactViewer.tsx index 3c4e2e01..2a4249d8 100644 --- a/optuna_dashboard/ts/components/ThreejsArtifactViewer.tsx +++ b/optuna_dashboard/ts/components/ThreejsArtifactViewer.tsx @@ -5,7 +5,9 @@ import { GizmoHelper, GizmoViewport, OrbitControls } from "@react-three/drei" import { STLLoader } from "three/examples/jsm/loaders/STLLoader" import { Rhino3dmLoader } from "three/examples/jsm/loaders/3DMLoader" import { PerspectiveCamera } from "three" -import { Modal, Box } from "@mui/material" +import { Modal, Box, useTheme } from "@mui/material" +import ClearIcon from "@mui/icons-material/Clear" +import IconButton from "@mui/material/IconButton" export const isThreejsArtifact = (artifact: Artifact): boolean => { return ( @@ -121,6 +123,7 @@ export const useThreejsArtifactModal = (): [ ] => { const [open, setOpen] = useState(false) const [target, setTarget] = useState<[string, Artifact | null]>(["", null]) + const theme = useTheme() const openModal = (artifactUrlPath: string, artifact: Artifact) => { setTarget([artifactUrlPath, artifact]) @@ -146,6 +149,19 @@ export const useThreejsArtifactModal = (): [ borderRadius: "15px", }} > + { + setOpen(false) + setTarget(["", null]) + }} + > + + =0.8.1", ] +preferential = [ + "botorch>=0.8.1", +] [project.scripts] optuna-dashboard = "optuna_dashboard._cli:main" diff --git a/python_tests/preferential/test_study.py b/python_tests/preferential/test_study.py index 7fc1aaba..9b4a3f31 100644 --- a/python_tests/preferential/test_study.py +++ b/python_tests/preferential/test_study.py @@ -3,6 +3,7 @@ from __future__ import annotations import copy import multiprocessing import pickle +import sys from typing import Callable from unittest.mock import patch import uuid @@ -22,6 +23,10 @@ from ..storage_supplier import parametrize_storages from ..storage_supplier import StorageSupplier +if sys.version_info < (3, 8): + pytest.skip("BoTorch dropped Python3.7 support", allow_module_level=True) + + @parametrize_storages def test_study_set_and_get_user_attrs(storage_supplier: Callable[[], StorageSupplier]) -> None: with storage_supplier() as storage: diff --git a/python_tests/test_api.py b/python_tests/test_api.py index f9c56e4f..bde7838a 100644 --- a/python_tests/test_api.py +++ b/python_tests/test_api.py @@ -1,6 +1,7 @@ from __future__ import annotations import json +import sys from unittest import TestCase import optuna @@ -14,6 +15,7 @@ from optuna_dashboard._preferential_history import remove_history from optuna_dashboard._preferential_history import report_history from optuna_dashboard._serializer import serialize_preference_history from optuna_dashboard.preferential import create_study +import pytest from .wsgi_client import send_request @@ -105,6 +107,7 @@ class APITestCase(TestCase): ) self.assertEqual(status, 400) + @pytest.mark.skipif(sys.version_info < (3, 8), reason="BoTorch dropped Python3.7 support") def test_get_best_trials_of_preferential_study(self) -> None: storage = optuna.storages.InMemoryStorage() study = create_study(n_generate=4, storage=storage) @@ -128,6 +131,7 @@ class APITestCase(TestCase): assert len(best_trials) == 1 assert best_trials[0]["number"] == 0 + @pytest.mark.skipif(sys.version_info < (3, 8), reason="BoTorch dropped Python3.7 support") def test_report_preference(self) -> None: storage = optuna.storages.InMemoryStorage() study = create_study(n_generate=4, storage=storage) @@ -161,6 +165,7 @@ class APITestCase(TestCase): assert better.number == 2 assert worse.number == 1 + @pytest.mark.skipif(sys.version_info < (3, 8), reason="BoTorch dropped Python3.7 support") def test_report_preference_when_typo_mode(self) -> None: storage = optuna.storages.InMemoryStorage() study = create_study(storage=storage, n_generate=3) @@ -184,6 +189,7 @@ class APITestCase(TestCase): ) self.assertEqual(status, 400) + @pytest.mark.skipif(sys.version_info < (3, 8), reason="BoTorch dropped Python3.7 support") def test_change_component(self) -> None: storage = optuna.storages.InMemoryStorage() study = create_study(storage=storage, n_generate=3) @@ -214,6 +220,7 @@ class APITestCase(TestCase): assert study_detail["feedback_component_type"]["output_type"] == "artifact" assert study_detail["feedback_component_type"]["artifact_key"] == "image" + @pytest.mark.skipif(sys.version_info < (3, 8), reason="BoTorch dropped Python3.7 support") def test_skip_trial(self) -> None: storage = optuna.storages.InMemoryStorage() study = create_study(n_generate=4, storage=storage) @@ -238,6 +245,7 @@ class APITestCase(TestCase): assert len(best_trials) == 1 assert best_trials[0].number == 2 + @pytest.mark.skipif(sys.version_info < (3, 8), reason="BoTorch dropped Python3.7 support") def test_remove_history(self) -> None: storage = optuna.storages.InMemoryStorage() study = create_study(storage=storage, n_generate=3) @@ -271,6 +279,7 @@ class APITestCase(TestCase): assert histories[0]["is_removed"] assert len(study.get_preferences()) == 0 + @pytest.mark.skipif(sys.version_info < (3, 8), reason="BoTorch dropped Python3.7 support") def test_restore_history(self) -> None: storage = optuna.storages.InMemoryStorage() study = create_study(storage=storage, n_generate=3) diff --git a/python_tests/test_preferential_history.py b/python_tests/test_preferential_history.py index 3d1a7d70..dcae4797 100644 --- a/python_tests/test_preferential_history.py +++ b/python_tests/test_preferential_history.py @@ -1,6 +1,7 @@ from __future__ import annotations import json +import sys from typing import Callable from typing import TYPE_CHECKING @@ -13,6 +14,7 @@ from optuna_dashboard._preferential_history import restore_history from optuna_dashboard._serializer import serialize_preference_history from optuna_dashboard.preferential import create_study from optuna_dashboard.preferential._system_attrs import _SYSTEM_ATTR_PREFIX_PREFERENCE +import pytest from .storage_supplier import parametrize_storages from .storage_supplier import StorageSupplier @@ -22,6 +24,10 @@ if TYPE_CHECKING: from optuna_dashboard._preferential_history import History +if sys.version_info < (3, 8): + pytest.skip("BoTorch dropped Python3.7 support", allow_module_level=True) + + @parametrize_storages def test_report_and_get_choices(storage_supplier: Callable[[], StorageSupplier]) -> None: with storage_supplier() as storage: diff --git a/python_tests/test_serializers.py b/python_tests/test_serializers.py index ea1c3517..0d1546d2 100644 --- a/python_tests/test_serializers.py +++ b/python_tests/test_serializers.py @@ -1,11 +1,14 @@ from __future__ import annotations +import sys + import optuna from optuna_dashboard._serializer import serialize_attrs from optuna_dashboard._serializer import serialize_study_detail from optuna_dashboard._serializer import serialize_study_summary from optuna_dashboard._storage import get_study_summaries from optuna_dashboard.preferential import create_study +import pytest def test_serialize_bytes() -> None: @@ -22,6 +25,7 @@ def test_serialize_dict() -> None: assert len(serialized) <= 1 +@pytest.mark.skipif(sys.version_info < (3, 8), reason="BoTorch dropped Python3.7 support") def test_get_study_detail_is_preferential() -> None: storage = optuna.storages.InMemoryStorage() study = create_study(n_generate=4, storage=storage) @@ -48,6 +52,7 @@ def test_get_study_detail_is_not_preferential() -> None: assert not study_detail["is_preferential"] +@pytest.mark.skipif(sys.version_info < (3, 8), reason="BoTorch dropped Python3.7 support") def test_get_study_summary_is_preferential() -> None: storage = optuna.storages.InMemoryStorage() create_study(n_generate=4, storage=storage) diff --git a/rustlib/src/lib.rs b/rustlib/src/lib.rs index ff09ef81..c326e503 100644 --- a/rustlib/src/lib.rs +++ b/rustlib/src/lib.rs @@ -1,16 +1,20 @@ +use fanova::{FanovaOptions, RandomForestOptions}; use js_sys::Array; use serde_wasm_bindgen::from_value; -use fanova::{FanovaOptions, RandomForestOptions}; use wasm_bindgen::prelude::*; #[wasm_bindgen] -pub fn wasm_fanova_calculate(features: Array, targets: Array) -> Vec { - // TODO(c-bata): Fix error handling +pub fn wasm_fanova_calculate(features: Array, targets: Array) -> Result, JsError> { let features_vec: Vec> = features .iter() - .map(|x| from_value::>(x).unwrap()) - .collect(); - let targets_vec: Vec = targets.iter().map(|x| x.as_f64().unwrap()).collect(); + .map(|x| from_value::>(x)) + .collect::>() + .map_err(|_| JsError::new("features must be of type number[][]"))?; + let targets_vec: Vec = targets + .iter() + .map(|x| x.as_f64()) + .collect::>() + .ok_or(JsError::new("targets must be of type number[]"))?; let mut fanova = FanovaOptions::new() .random_forest(RandomForestOptions::new().seed(0)) @@ -18,9 +22,10 @@ pub fn wasm_fanova_calculate(features: Array, targets: Array) -> Vec { features_vec.iter().map(|x| x.as_slice()).collect(), &targets_vec, ) - .unwrap(); + .map_err(|e| JsError::new(&format!("failed to build fANOVA model: {}", e)))?; let importances = (0..features_vec.len()) .map(|i| fanova.quantify_importance(&[i]).mean) .collect::>(); - return importances; + + Ok(importances) } diff --git a/standalone_app/src/components/PlotImportance.tsx b/standalone_app/src/components/PlotImportance.tsx index 24e9cd8f..7303b2db 100644 --- a/standalone_app/src/components/PlotImportance.tsx +++ b/standalone_app/src/components/PlotImportance.tsx @@ -39,6 +39,7 @@ export const PlotImportance: FC<{ study: Study }> = ({ study }) => { const values = filteredTrials.map( (t) => t.values?.[objectiveId] as number ) + // TODO: handle errors thrown by wasm_fanova_calculate const importance = wasm_fanova_calculate(features, values) return study.intersection_search_space.map((s, i) => ({ name: s.name, diff --git a/vscode/README.md b/vscode/README.md index 78587c37..960669a3 100644 --- a/vscode/README.md +++ b/vscode/README.md @@ -14,6 +14,10 @@ Nothing to configure. ## Release Notes +### 0.1.0 + +Added older database schemas support (Optuna 2.6.0 or later) + ### 0.0.1 Initial Release diff --git a/vscode/package.json b/vscode/package.json index a5c1d818..6a99a905 100644 --- a/vscode/package.json +++ b/vscode/package.json @@ -3,7 +3,7 @@ "displayName": "Optuna Dashboard", "description": "Web Dashboard for Optuna", "publisher": "Optuna", - "version": "0.0.1", + "version": "0.1.0", "license": "MIT", "icon": "images/optuna-logo.png", "engines": {