From 92b94b64888d9b6b805b169ec2cc9e7994ee65e4 Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Mon, 11 Sep 2023 14:40:57 +0900 Subject: [PATCH] merge and modify --- .github/workflows/python-coverage.yml | 2 +- docs/api.rst | 1 + .../preferential-optimization/generator.py | 3 - optuna_dashboard/__init__.py | 3 +- optuna_dashboard/_app.py | 40 +- optuna_dashboard/_custom_plot_data.py | 134 ++++ optuna_dashboard/_preferential_history.py | 80 +++ optuna_dashboard/_serializer.py | 35 ++ optuna_dashboard/preferential/_study.py | 94 +-- .../preferential/_system_attrs.py | 27 +- optuna_dashboard/preferential/samplers/gp.py | 579 ++++++++---------- optuna_dashboard/ts/action.ts | 6 +- optuna_dashboard/ts/apiClient.ts | 39 +- optuna_dashboard/ts/components/App.tsx | 9 + optuna_dashboard/ts/components/AppDrawer.tsx | 31 +- .../ts/components/ArtifactCardMedia.tsx | 41 ++ .../ts/components/PreferenceHistory.tsx | 218 +++++++ .../ts/components/PreferentialTrials.tsx | 473 +++++++------- .../ts/components/StudyDetail.tsx | 3 + .../ts/components/StudyHistory.tsx | 11 + .../ts/components/ThreejsArtifactViewer.tsx | 54 +- .../ts/components/TrialArtifactCards.tsx | 233 +++++++ optuna_dashboard/ts/components/TrialList.tsx | 360 +---------- .../ts/components/UserDefinedPlot.tsx | 21 + optuna_dashboard/ts/types/index.d.ts | 17 + pyproject.toml | 1 + python_tests/preferential/test_study.py | 6 +- .../preferential/test_system_attrs.py | 7 +- python_tests/test_api.py | 57 +- python_tests/test_custom_plot_data.py | 86 +++ python_tests/test_preferential_history.py | 61 ++ python_tests/test_serializers.py | 4 +- 32 files changed, 1755 insertions(+), 981 deletions(-) create mode 100644 optuna_dashboard/_custom_plot_data.py create mode 100644 optuna_dashboard/_preferential_history.py create mode 100644 optuna_dashboard/ts/components/ArtifactCardMedia.tsx create mode 100644 optuna_dashboard/ts/components/PreferenceHistory.tsx create mode 100644 optuna_dashboard/ts/components/TrialArtifactCards.tsx create mode 100644 optuna_dashboard/ts/components/UserDefinedPlot.tsx create mode 100644 python_tests/test_custom_plot_data.py create mode 100644 python_tests/test_preferential_history.py diff --git a/.github/workflows/python-coverage.yml b/.github/workflows/python-coverage.yml index 4bda6350..c96f506b 100644 --- a/.github/workflows/python-coverage.yml +++ b/.github/workflows/python-coverage.yml @@ -45,4 +45,4 @@ jobs: with: token: ${{ secrets.CODECOV_TOKEN }} file: ./coverage.xml - fail_ci_if_error: true + fail_ci_if_error: false diff --git a/docs/api.rst b/docs/api.rst index 09a9c14b..aadd2718 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -14,6 +14,7 @@ General APIs optuna_dashboard.wsgi optuna_dashboard.set_objective_names optuna_dashboard.save_note + optuna_dashboard.save_plotly_graph_object Human-in-the-loop ----------------- diff --git a/examples/preferential-optimization/generator.py b/examples/preferential-optimization/generator.py index e94f1d05..d8f7d3bd 100644 --- a/examples/preferential-optimization/generator.py +++ b/examples/preferential-optimization/generator.py @@ -64,9 +64,6 @@ def main() -> NoReturn: ) save_note(trial, note) - # 5. Mark comparison ready - study.mark_comparison_ready(trial) - if __name__ == "__main__": main() diff --git a/optuna_dashboard/__init__.py b/optuna_dashboard/__init__.py index 3d363cf4..5bb3f301 100644 --- a/optuna_dashboard/__init__.py +++ b/optuna_dashboard/__init__.py @@ -1,5 +1,6 @@ from ._app import run_server # noqa from ._app import wsgi # noqa +from ._custom_plot_data import save_plotly_graph_object # noqa from ._form_widget import ChoiceWidget # noqa from ._form_widget import dict_to_form_widget # noqa from ._form_widget import ObjectiveChoiceWidget # noqa @@ -15,4 +16,4 @@ from ._note import get_note # noqa from ._note import save_note # noqa -__version__ = "0.12.0" +__version__ = "0.13.0b1" diff --git a/optuna_dashboard/_app.py b/optuna_dashboard/_app.py index 70acc13f..fad4b74c 100644 --- a/optuna_dashboard/_app.py +++ b/optuna_dashboard/_app.py @@ -25,9 +25,12 @@ from . import _note as note from ._bottle_util import BottleViewReturn from ._bottle_util import json_api_view from ._cached_extra_study_property import get_cached_extra_study_property +from ._custom_plot_data import get_plotly_graph_objects from ._importance import get_param_importance_from_trials_cache from ._pareto_front import get_pareto_front_trials from ._preference_setting import _register_output_component +from ._preferential_history import NewHistory +from ._preferential_history import report_history from ._rdb_migration import register_rdb_migration_route from ._serializer import serialize_study_detail from ._serializer import serialize_study_summary @@ -41,7 +44,6 @@ from .artifact._backend import register_artifact_route from .artifact._backend_to_store import to_artifact_store from .preferential._study import _SYSTEM_ATTR_PREFERENTIAL_STUDY from .preferential._study import get_best_trials as get_best_preferential_trials -from .preferential._system_attrs import report_preferences from .preferential._system_attrs import report_skip @@ -214,6 +216,8 @@ def create_app( union_user_attrs, has_intermediate_values, ) = get_cached_extra_study_property(study_id, trials) + + plotly_graph_objects = get_plotly_graph_objects(system_attrs) return serialize_study_detail( summary, best_trials, @@ -222,6 +226,7 @@ def create_app( union, union_user_attrs, has_intermediate_values, + plotly_graph_objects, ) @app.get("/api/studies//param_importances") @@ -270,17 +275,34 @@ def create_app( @json_api_view def post_preference(study_id: int) -> dict[str, Any]: try: - best_trials = [int(d) for d in request.json.get("best_trials", [])] - worst_trials = [int(d) for d in request.json.get("worst_trials", [])] + mode = request.json.get("mode", "") + candidates = [int(d) for d in request.json.get("candidates", [])] + clicked = int(request.json.get("clicked", -1)) except ValueError: response.status = 400 - return {"reason": "best_trials and worst_trials must be an array of integers."} - if len(best_trials) == 0 or len(worst_trials) == 0: - response.status = 400 # Bad request - return {"reason": "You need to set best_trials and worst_trials"} + return { + "reason": ( + "`candidates` should be an array of integers and " + "`clicked` should be an integer." + ) + } - preferences = [(best, worst) for best in best_trials for worst in worst_trials] - report_preferences(study_id, storage, preferences) + if clicked == -1: + response.status = 400 + return {"reason": "`clicked` should be specified."} + if mode != "ChooseWorst": + response.status = 400 + return {"reason": "`mode` should be 'ChooseWorst'."} + + report_history( + study_id, + storage, + NewHistory( + mode=mode, + candidates=candidates, + clicked=clicked, + ), + ) response.status = 204 return {} diff --git a/optuna_dashboard/_custom_plot_data.py b/optuna_dashboard/_custom_plot_data.py new file mode 100644 index 00000000..a4dfa4af --- /dev/null +++ b/optuna_dashboard/_custom_plot_data.py @@ -0,0 +1,134 @@ +from __future__ import annotations + +import math +from typing import TYPE_CHECKING +import uuid + +from optuna import Study + + +if TYPE_CHECKING: + from typing import Any + + from optuna.storages import BaseStorage + import plotly.graph_objs as go + + +SYSTEM_ATTR_PLOT_DATA = "dashboard:plot_data:" +SYSTEM_ATTR_MAX_LENGTH = 2045 + + +def save_plotly_graph_object( + study: Study, figure: go.Figure, *, graph_object_id: str | None = None +) -> str: + """Save the user-defined plotly's graph object to the study. + + Example: + + .. code-block:: python + + import optuna + from optuna_dashboard import save_plotly_graph_object + + def objective(trial): + x = trial.suggest_float("x", -100, 100) + y = trial.suggest_categorical("y", [-1, 0, 1]) + return x**2 + y + + study = optuna.create_study() + study.optimize(objective, n_trials=100) + + figure = optuna.visualization.plot_optimization_history(study) + save_plotly_graph_object(study, figure) + + Args: + study: + Target study object. + plot_data: + The plotly's graph object to save. + graph_object_id: + Unique identifier of the graph object. If specified, the graph object is overwritten. + This must be a valid HTML id attribute value. + + Returns: + The graph object ID. + """ + if graph_object_id is not None and not is_valid_graph_object_id(graph_object_id): + raise ValueError("graph_object_id must be a valid HTML id attribute value.") + + storage = study._storage + study_id = study._study_id + + graph_object_id = graph_object_id or str(uuid.uuid4()) + key = SYSTEM_ATTR_PLOT_DATA + graph_object_id + ":" + plot_data_json_str = figure.to_json() + save_graph_object_json(storage, study_id, key, plot_data_json_str) + return graph_object_id + + +def save_graph_object_json( + storage: BaseStorage, study_id: int, key_prefix: str, plot_data_json_str: str +) -> None: + plot_data_system_attrs = split_plot_data(plot_data_json_str, key_prefix) + for k, v in plot_data_system_attrs.items(): + storage.set_study_system_attr(study_id, k, v) + + # Clear previous graph object attributes + study_system_attrs = storage.get_study_system_attrs(study_id) + all_plot_data_system_attrs = [k for k in study_system_attrs if k.startswith(key_prefix)] + if len(all_plot_data_system_attrs) > len(plot_data_system_attrs): + for i in range(len(plot_data_system_attrs), len(all_plot_data_system_attrs)): + storage.set_study_system_attr(study_id, f"{key_prefix}{i}", "") + + +def list_graph_object_ids(system_attrs: dict[str, Any]) -> list[str]: + titles = set() + for key in system_attrs: + if not key.startswith(SYSTEM_ATTR_PLOT_DATA): + continue + + s = key.split(":", maxsplit=2) # e.g. ["dashboard", "plot_data", "Optimization History:1"] + if len(s) != 3: + continue + # Please note that title may contain ":". + title = s[2].rsplit(":", maxsplit=1)[0] + titles.add(title) + return list(titles) + + +def get_plotly_graph_objects(system_attrs: dict[str, Any]) -> dict[str, str]: + graph_objects = {} + for title in list_graph_object_ids(system_attrs): + key_prefix = SYSTEM_ATTR_PLOT_DATA + title + ":" + plot_data_attrs = {k: v for k, v in system_attrs.items() if k.startswith(key_prefix)} + graph_objects[title] = concat_plot_data(plot_data_attrs, key_prefix) + return graph_objects + + +def split_plot_data(plot_data_str: str, key_prefix: str) -> dict[str, str]: + plot_data_len = len(plot_data_str) + attrs = {} + for i in range(math.ceil(plot_data_len / SYSTEM_ATTR_MAX_LENGTH)): + start = i * SYSTEM_ATTR_MAX_LENGTH + end = min((i + 1) * SYSTEM_ATTR_MAX_LENGTH, plot_data_len) + attrs[f"{key_prefix}{i}"] = plot_data_str[start:end] + return attrs + + +def concat_plot_data(plot_data_attrs: dict[str, str], key_prefix: str) -> str: + return "".join(plot_data_attrs[f"{key_prefix}{i}"] for i in range(len(plot_data_attrs))) + + +def is_valid_graph_object_id(graph_object_id: str) -> bool: + if len(graph_object_id) == 0: + return False + + # Can only contain letters [A-Za-z], numbers [0-9], hyphens ("-"), underscores ("_"), + # colons, and periods. + if not all( + "a" <= c <= "z" or "A" <= c <= "Z" or "0" <= c <= "9" or c in ("-", "_", ":", ".") + for c in graph_object_id[1:] + ): + return False + # Unlike HTML id attribute, graph object id can begin with a letter [A-Za-z] + return True diff --git a/optuna_dashboard/_preferential_history.py b/optuna_dashboard/_preferential_history.py new file mode 100644 index 00000000..6b81b9bb --- /dev/null +++ b/optuna_dashboard/_preferential_history.py @@ -0,0 +1,80 @@ +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime +import json +from typing import TYPE_CHECKING +import uuid + +from optuna.storages import BaseStorage + +from .preferential._system_attrs import report_preferences + + +_SYSTEM_ATTR_PREFIX_HISTORY = "preference:history" + +if TYPE_CHECKING: + from typing import Literal + from typing import TypedDict + + FeedbackMode = Literal["ChooseWorst"] + ChooseWorstHistory = TypedDict( + "ChooseWorstHistory", + { + "mode": FeedbackMode, + "id": str, + "preference_id": str, + "timestamp": str, + "candidates": list[int], + "clicked": int, + }, + ) + History = ChooseWorstHistory + + +@dataclass +class NewHistory: + mode: FeedbackMode + candidates: list[int] + clicked: int + + +def report_history( + study_id: int, + storage: BaseStorage, + input_data: NewHistory, +) -> None: + preferences = [] + # TODO(moririn): Use TypeGuard after adding other history types. + if input_data.mode == "ChooseWorst": + preferences = [ + (best, input_data.clicked) + for best in input_data.candidates + if best != input_data.clicked + ] + else: + assert False, f"Unknown data: {input_data}" + + preference_id = report_preferences( + study_id=study_id, + storage=storage, + preferences=preferences, + ) + history_id = str(uuid.uuid4()) + + if input_data.mode == "ChooseWorst": + history: ChooseWorstHistory = { + "mode": "ChooseWorst", + "id": history_id, + "preference_id": preference_id, + "timestamp": datetime.now().isoformat(), + "candidates": input_data.candidates, + "clicked": input_data.clicked, + } + + key = _SYSTEM_ATTR_PREFIX_HISTORY + history_id + storage.set_study_system_attr( + study_id=study_id, + key=key, + value=json.dumps(history), + ) diff --git a/optuna_dashboard/_serializer.py b/optuna_dashboard/_serializer.py index b2494068..1f49e925 100644 --- a/optuna_dashboard/_serializer.py +++ b/optuna_dashboard/_serializer.py @@ -1,5 +1,6 @@ from __future__ import annotations +from datetime import datetime import json from typing import Any from typing import TYPE_CHECKING @@ -16,6 +17,7 @@ from ._form_widget import get_form_widgets_json from ._named_objectives import get_objective_names from ._preference_setting import _SYSTEM_ATTR_FEEDBACK_ARTIFACT_KEY from ._preference_setting import _SYSTEM_ATTR_FEEDBACK_COMPONENT_TYPE +from ._preferential_history import _SYSTEM_ATTR_PREFIX_HISTORY from .artifact._backend import list_trial_artifacts from .preferential._study import _SYSTEM_ATTR_PREFERENTIAL_STUDY @@ -24,6 +26,9 @@ if TYPE_CHECKING: from typing import Literal from typing import TypedDict + from ._preferential_history import ChooseWorstHistory + from ._preferential_history import History + Attribute = TypedDict( "Attribute", { @@ -129,6 +134,7 @@ def serialize_study_detail( union: list[tuple[str, BaseDistribution]], union_user_attrs: list[tuple[str, bool]], has_intermediate_values: bool, + plotly_graph_objects: dict[str, str], ) -> dict[str, Any]: serialized: dict[str, Any] = { "name": summary.study_name, @@ -161,9 +167,38 @@ def serialize_study_detail( serialized["feedback_component_type"] = system_attrs[_SYSTEM_ATTR_FEEDBACK_COMPONENT_TYPE] if _SYSTEM_ATTR_FEEDBACK_ARTIFACT_KEY in system_attrs: serialized["feedback_artifact_key"] = system_attrs[_SYSTEM_ATTR_FEEDBACK_ARTIFACT_KEY] + if serialized["is_preferential"]: + serialized["preference_history"] = serialize_preference_history(system_attrs) + serialized["plotly_graph_objects"] = [ + {"id": id_, "graph_object": graph_object} + for id_, graph_object in plotly_graph_objects.items() + ] return serialized +def serialize_preference_history( + system_attrs: dict[str, Any], +) -> list[History]: + histories: list[History] = [] + for k, v in system_attrs.items(): + if not k.startswith(_SYSTEM_ATTR_PREFIX_HISTORY): + continue + choice: dict[str, Any] = json.loads(v) + if choice["mode"] == "ChooseWorst": + history: ChooseWorstHistory = { + "mode": "ChooseWorst", + "id": choice["id"], + "preference_id": choice["preference_id"], + "timestamp": choice["timestamp"], + "candidates": choice["candidates"], + "clicked": choice["clicked"], + } + histories.append(history) + + histories.sort(key=lambda c: datetime.fromisoformat(c["timestamp"])) + return histories + + def serialize_frozen_trial( study_id: int, trial: FrozenTrial, study_system_attrs: dict[str, Any] ) -> dict[str, Any]: diff --git a/optuna_dashboard/preferential/_study.py b/optuna_dashboard/preferential/_study.py index ce691f42..6e093683 100644 --- a/optuna_dashboard/preferential/_study.py +++ b/optuna_dashboard/preferential/_study.py @@ -14,6 +14,7 @@ from optuna.trial import FrozenTrial from optuna.trial import TrialState from optuna_dashboard.preferential._system_attrs import get_n_generate from optuna_dashboard.preferential._system_attrs import get_preferences +from optuna_dashboard.preferential._system_attrs import get_skipped_trial_ids from optuna_dashboard.preferential._system_attrs import is_skipped_trial from optuna_dashboard.preferential._system_attrs import report_preferences from optuna_dashboard.preferential._system_attrs import set_n_generate @@ -21,7 +22,6 @@ from optuna_dashboard.preferential._system_attrs import set_n_generate _logger = logging.get_logger(__name__) _SYSTEM_ATTR_PREFERENTIAL_STUDY = "preference:is_preferential" -_SYSTEM_ATTR_COMPARISON_READY = "preference:comparison_ready" class PreferentialStudy: @@ -62,13 +62,6 @@ class PreferentialStudy: def best_trials(self) -> list[FrozenTrial]: """Return the trials that is not dominated by other trials. - .. seealso:: - - See `Study.best_trials`_ for details. - - .. _Study.best_trials: https://optuna.readthedocs.io/en/stable/reference/\ - generated/optuna.study.Study.html#optuna.study.Study.best_trials - Returns: A list of FrozenTrial object """ @@ -182,6 +175,38 @@ class PreferentialStudy: """ self._study.add_trials(trials) + def enqueue_trial( + self, + params: dict[str, Any], + user_attrs: dict[str, Any] | None = None, + skip_if_exists: bool = False, + ) -> None: + """Enqueue a trial with given parameter values. + + You can fix the next sampling parameters which will be evaluated in your + objective function. + + .. seealso:: + + See `Study.enqueue_trials`_ for details. + + .. _Study.get_trials: https://optuna.readthedocs.io/en/stable/reference/\ + generated/optuna.study.Study.html#optuna.study.Study.enqueue_trials + + Args: + params: + Parameter values to pass your objective function. + user_attrs: + A dictionary of user-specific attributes other than ``params``. + skip_if_exists: + When :obj:`True`, prevents duplicate trials from being enqueued again. + + .. note:: + This method might produce duplicated trials if called simultaneously + by multiple processes at the same time with same ``params`` dict. + """ + self._study.enqueue_trial(params, user_attrs, skip_if_exists) + def report_preference( self, better_trials: FrozenTrial | list[FrozenTrial], @@ -219,8 +244,11 @@ class PreferentialStudy: Returns: A list of the pair of FrozenTrial objects. The left trial is better than the right one. """ + + preferences = get_preferences( + self._study._storage.get_study_system_attrs(self._study._study_id) + ) # Must come before study.get_trials() trials = self._study.get_trials(deepcopy=deepcopy) - preferences = get_preferences(self._study._study_id, self._study._storage) return [(trials[better], trials[worse]) for (better, worse) in preferences] def set_user_attr(self, key: str, value: Any) -> None: @@ -237,24 +265,6 @@ class PreferentialStudy: """ self._study.set_user_attr(key, value) - def mark_comparison_ready(self, trial_or_number: optuna.Trial | int) -> None: - """Mark trials ready to compare. - - Args: - trial_or_number: - A Trial object or trial_number. - """ - storage = self._study._storage - if isinstance(trial_or_number, optuna.Trial): - trial_id = trial_or_number._trial_id - elif isinstance(trial_or_number, int): - trial_id = storage.get_trial_id_from_study_id_trial_number( - self._study._study_id, trial_or_number - ) - else: - raise RuntimeError("Unexpected trial type") - storage.set_trial_system_attr(trial_id, _SYSTEM_ATTR_COMPARISON_READY, True) - def should_generate(self) -> bool: """Return whether the generator should generate a new trial now. @@ -263,21 +273,33 @@ class PreferentialStudy: to generate a new trial if this method returns :obj:`True`, and to wait for human evaluation if this method returns :obj:`False`. """ - return len(self.best_trials) < get_n_generate(self._study.system_attrs) + study_system_attrs = self._study._storage.get_study_system_attrs( + self._study._study_id + ) # Must come before _study.get_trials() + trials = self._study.get_trials( + deepcopy=False, states=(TrialState.COMPLETE, TrialState.RUNNING) + ) + worse_trial_numbers = {worse for _, worse in get_preferences(study_system_attrs)} + skipped_trial_ids = set(get_skipped_trial_ids(study_system_attrs)) + active_trials = [ + t + for t in trials + if t.number not in worse_trial_numbers and t._trial_id not in skipped_trial_ids + ] + return len(active_trials) < get_n_generate(self._study.system_attrs) def get_best_trials(study_id: int, storage: optuna.storages.BaseStorage) -> list[FrozenTrial]: - preferences = get_preferences(study_id, storage) + preferences = get_preferences(storage.get_study_system_attrs(study_id)) worse_numbers = {worse for _, worse in preferences} + nondominated_numbers = {better for better, _ in preferences if better not in worse_numbers} + trials = storage.get_all_trials(study_id, deepcopy=False) + study_system_attrs = storage.get_study_system_attrs(study_id) + best_trials = [] - for t in storage.get_all_trials( - study_id, deepcopy=False, states=(TrialState.COMPLETE, TrialState.RUNNING) - ): - if not t.system_attrs.get(_SYSTEM_ATTR_COMPARISON_READY, False): - continue - if t.number in worse_numbers: - continue + for n in nondominated_numbers: + t = trials[n] if is_skipped_trial(t._trial_id, study_system_attrs): continue best_trials.append(copy.deepcopy(t)) diff --git a/optuna_dashboard/preferential/_system_attrs.py b/optuna_dashboard/preferential/_system_attrs.py index 2964c9e0..47c2a486 100644 --- a/optuna_dashboard/preferential/_system_attrs.py +++ b/optuna_dashboard/preferential/_system_attrs.py @@ -16,8 +16,9 @@ def report_preferences( study_id: int, storage: BaseStorage, preferences: list[tuple[int, int]], -) -> None: - key = _SYSTEM_ATTR_PREFIX_PREFERENCE + str(uuid.uuid4()) +) -> str: + preference_id = str(uuid.uuid4()) + key = _SYSTEM_ATTR_PREFIX_PREFERENCE + preference_id storage.set_study_system_attr( study_id=study_id, key=key, @@ -31,15 +32,12 @@ def report_preferences( trial_id = trials[number]._trial_id if trials[number].state != TrialState.COMPLETE: storage.set_trial_state_values(trial_id, TrialState.COMPLETE, values) + return preference_id -def get_preferences( - study_id: int, - storage: BaseStorage, -) -> list[tuple[int, int]]: +def get_preferences(study_system_attrs: dict[str, Any]) -> list[tuple[int, int]]: preferences: list[tuple[int, int]] = [] - system_attrs = storage.get_study_system_attrs(study_id) - for k, v in system_attrs.items(): + for k, v in study_system_attrs.items(): if not k.startswith(_SYSTEM_ATTR_PREFIX_PREFERENCE): continue preferences.extend(v) # type: ignore @@ -63,6 +61,19 @@ def is_skipped_trial(trial_id: int, study_system_attrs: dict[str, Any]) -> bool: return key in study_system_attrs +def get_skipped_trial_ids(study_system_attrs: dict[str, Any]) -> list[int]: + skipped_trial_ids: list[int] = [] + for k in study_system_attrs: + if not k.startswith(_SYSTEM_ATTR_PREFIX_SKIP_TRIAL): + continue + try: + trial_id = int(k[len(_SYSTEM_ATTR_PREFIX_SKIP_TRIAL) :]) # noqa: E203 + skipped_trial_ids.append(trial_id) + except ValueError: + continue + return skipped_trial_ids + + def get_n_generate(study_system_attrs: dict[str, Any]) -> int: return study_system_attrs[_SYSTEM_ATTR_N_GENERATE] diff --git a/optuna_dashboard/preferential/samplers/gp.py b/optuna_dashboard/preferential/samplers/gp.py index ff4001c1..8349b1a4 100644 --- a/optuna_dashboard/preferential/samplers/gp.py +++ b/optuna_dashboard/preferential/samplers/gp.py @@ -1,155 +1,41 @@ from __future__ import annotations import math -from math import erfc from typing import Any +from typing import Callable -from botorch.acquisition.analytic import LogExpectedImprovement -from botorch.models.gpytorch import GPyTorchModel -from botorch.optim import optimize_acqf +import botorch.acquisition.analytic +import botorch.models.model +import botorch.optim +import botorch.posteriors.gpytorch import gpytorch.constraints import gpytorch.kernels -import gpytorch.likelihoods.gaussian_likelihood -from gpytorch.likelihoods.gaussian_likelihood import GaussianLikelihood -from gpytorch.likelihoods.gaussian_likelihood import Interval from gpytorch.likelihoods.gaussian_likelihood import Prior -from gpytorch.models.exact_gp import ExactGP -import gpytorch.module -from linear_operator.operators import DiagLinearOperator -from linear_operator.operators import LinearOperator -from linear_operator.utils.errors import NotPSDError import numpy as np import optuna -from optuna import distributions -from optuna import Study -from optuna._transform import _SearchSpaceTransform -from optuna.distributions import BaseDistribution -from optuna.search_space import IntersectionSearchSpace -from optuna.trial import FrozenTrial -import pyro -import pyro.infer.autoguide -import pyro.infer.mcmc -from scipy.special import erfcinv +import optuna._transform import torch from torch import Tensor from .._system_attrs import get_preferences -class _WeightedGaussianLikelihood(GaussianLikelihood): - def __init__( - self, - weights: torch.Tensor | None = None, - noise_prior: Prior | None = None, - noise_constraint: Interval | None = None, - batch_shape: torch.Size = torch.Size(), - **kwargs: Any, - ) -> None: - super().__init__( - noise_prior=noise_prior, - noise_constraint=noise_constraint, - batch_shape=batch_shape, - **kwargs, - ) - self.weights = weights - - def _shaped_noise_covar( - self, base_shape: torch.Size, *params: Any, **kwargs: Any - ) -> Tensor | LinearOperator: - assert self.weights is not None - assert base_shape[-1] == self.weights.shape[-1] - return DiagLinearOperator(1.0 / self.weights) * super()._shaped_noise_covar( - base_shape, *params, **kwargs - ) - - -def _sample_y( - preferences: np.ndarray, - cov_X_X: np.ndarray, - obs_noise_var: float, - cycles: int, - initial_sample: np.ndarray, - rng: np.random.RandomState, -) -> np.ndarray: - # TODO: Refactor and write tests for this function. - - N = cov_X_X.shape[0] - M = len(preferences) - cov_X_X = cov_X_X + np.eye(N) * 1e-6 # Add jitter - cov_X_X_chol = np.linalg.cholesky(cov_X_X) - cov_X_X_inv = np.linalg.inv(cov_X_X) - - # (sI + A K A^T)^-1 = s^-1 I - s^-2 A(K^-1 + s^-1 A^T A)^-1 A^T - - schur = cov_X_X_inv.copy() - np.add.at(schur, (preferences[:, 0], preferences[:, 0]), 1.0 / (2 * obs_noise_var)) - np.add.at(schur, (preferences[:, 1], preferences[:, 1]), 1.0 / (2 * obs_noise_var)) - np.add.at(schur, (preferences[:, 0], preferences[:, 1]), -1.0 / (2 * obs_noise_var)) - np.add.at(schur, (preferences[:, 1], preferences[:, 0]), -1.0 / (2 * obs_noise_var)) - idx_M = np.arange(M) - - schur_inv = np.linalg.inv(schur) - - cov_diff_inv = schur_inv[:, preferences[:, 0]] - schur_inv[:, preferences[:, 1]] - cov_diff_inv = cov_diff_inv[preferences[:, 0], :] - cov_diff_inv[preferences[:, 1], :] - cov_diff_inv *= -1 / (2 * obs_noise_var) ** 2 - cov_diff_inv[idx_M, idx_M] += 1.0 / (2 * obs_noise_var) - - diffs = _orthants_MVN_Gibbs_sampling( - cov_diff_inv, - cycles=cycles, - initial_sample=initial_sample[:, 0] - initial_sample[:, 1], - rng=rng, - )[-1] - - random_ys = (cov_X_X_chol @ rng.randn(N))[preferences] + np.sqrt(obs_noise_var) * rng.randn( - M, 2 - ) - errors = diffs - (random_ys[:, 0] - random_ys[:, 1]) - cov_diff_inv_errors = cov_diff_inv @ errors - - AT_cov_diff_inv_errors = np.zeros((N,)) - np.add.at(AT_cov_diff_inv_errors, preferences[:, 0], cov_diff_inv_errors) - np.add.at(AT_cov_diff_inv_errors, preferences[:, 1], -cov_diff_inv_errors) - - return ( - random_ys - + (cov_X_X @ AT_cov_diff_inv_errors)[preferences] - + obs_noise_var * np.array([[1, -1]]) * cov_diff_inv_errors[:, None] - ) - - -_SQRT2 = math.sqrt(2) - - -def _orthants_MVN_Gibbs_sampling( - cov_inv: np.ndarray, - cycles: int, - initial_sample: np.ndarray, - rng: np.random.RandomState, -) -> np.ndarray: +def _orthants_MVN_Gibbs_sampling(cov_inv: Tensor, cycles: int, initial_sample: Tensor) -> Tensor: dim = cov_inv.shape[0] assert cov_inv.shape == (dim, dim) - if initial_sample is None: - sample_chain = np.zeros(dim) - else: - sample_chain = initial_sample + sample_chain = initial_sample + conditional_std = torch.rsqrt(torch.diag(cov_inv)) + scaled_cov_inv = cov_inv / torch.diag(cov_inv)[:, None] - conditional_std = 1 / np.sqrt(np.diag(cov_inv)) - - scaled_cov_inv = cov_inv / np.c_[np.diag(cov_inv)] - - out = np.empty((cycles + 1, dim)) + out = torch.empty((cycles + 1, dim), dtype=torch.float64) out[0, :] = sample_chain for i in range(cycles): for j in range(dim): conditional_mean = sample_chain[j] - scaled_cov_inv[j] @ sample_chain sample_chain[j] = ( - _one_side_trunc_norm_sampling( - lower=-conditional_mean / conditional_std[j], rng=rng - ) + _one_side_trunc_norm_sampling(lower=-conditional_mean / conditional_std[j]) * conditional_std[j] + conditional_mean ) @@ -158,144 +44,234 @@ def _orthants_MVN_Gibbs_sampling( return out -def _one_side_trunc_norm_sampling(lower: float, rng: np.random.RandomState) -> float: - return erfcinv(rng.rand() * erfc(lower / _SQRT2)) * _SQRT2 +def _one_side_trunc_norm_sampling(lower: Tensor) -> Tensor: + if lower > 4.0: + r = torch.clamp_min(torch.rand(torch.Size(()), dtype=torch.float64), min=1e-300) + return (lower * lower - 2 * r.log()).sqrt() + else: + SQRT2 = math.sqrt(2) + r = torch.rand(torch.Size(()), dtype=torch.float64) * torch.erfc(lower / SQRT2) + while 1 - r == 1: + r = torch.rand(torch.Size(()), dtype=torch.float64) * torch.erfc(lower / SQRT2) + return torch.erfinv(1 - r) * SQRT2 -class _PreferentialGP(GPyTorchModel, ExactGP): - _num_outputs = 1 +_orthants_MVN_Gibbs_sampling_jit = torch.jit.script(_orthants_MVN_Gibbs_sampling) + +def _compute_cov_diff_diff_inv(preferences: Tensor, cov_x_x: Tensor, noise_var: Tensor) -> Tensor: + N = cov_x_x.shape[0] + M = preferences.shape[0] + + # (sI + A K A^T)^-1 = s^-1 I - s^-2 A(K^-1 + s^-1 A^T A)^-1 A^T + # (K^-1 + s^-1 A^T A)^-1 = K (I + s^-1 A^T A K)^-1 (To avoid computing K^-1) + + I_plus_sinv_AT_A_K = torch.eye(N, dtype=torch.float64) + A_K = cov_x_x[preferences[:, 0], :] - cov_x_x[preferences[:, 1], :] + I_plus_sinv_AT_A_K.index_add_(0, preferences[:, 0], A_K * (1 / noise_var)) + I_plus_sinv_AT_A_K.index_add_(0, preferences[:, 1], A_K * (-1 / noise_var)) + schur_inv: Tensor = torch.linalg.solve(I_plus_sinv_AT_A_K, cov_x_x, left=False) + cov_diff_diff_inv = schur_inv[:, preferences[:, 0]] - schur_inv[:, preferences[:, 1]] + cov_diff_diff_inv = ( + cov_diff_diff_inv[preferences[:, 0], :] - cov_diff_diff_inv[preferences[:, 1], :] + ) + cov_diff_diff_inv *= -1 / noise_var**2 + idx_M = torch.arange(M) + cov_diff_diff_inv[idx_M, idx_M] += 1.0 / noise_var + + return cov_diff_diff_inv + + +class _SampledGP(botorch.models.model.Model): def __init__( self, - kernel: gpytorch.kernels.Kernel, - noise_prior: Prior | None = None, - noise_constraint: Interval | None = None, + kernel_func: Callable[[Tensor, Tensor], Tensor], + x: Tensor, + preferences: Tensor, + noise_var: Tensor, + diff: Tensor, ) -> None: - GPyTorchModel.__init__(self) - likelihood = _WeightedGaussianLikelihood( - noise_prior=noise_prior, noise_constraint=noise_constraint + super().__init__() + self.kernel_func = kernel_func + self.x = x + self.preferences = preferences + self.diff = diff + self.noise_var = noise_var + self._cov_diff_diff_inv = _compute_cov_diff_diff_inv( + preferences=preferences, + cov_x_x=self.kernel_func(x, x), + noise_var=noise_var, ) - ExactGP.__init__(self, train_inputs=None, train_targets=None, likelihood=likelihood) - self.covar_module = kernel - self._last_params: dict[str, torch.Tensor] | None = None - self._last_mcmc_step_size: float | None = None + def posterior( + self, + X: Tensor, + output_indices: list[int] | None = None, + observation_noise: bool = False, + posterior_transform: Any | None = None, + **kwargs: Any, + ) -> botorch.posteriors.gpytorch.GPyTorchPosterior: + assert posterior_transform is None + assert output_indices is None + assert self.x.shape[-1] == X.shape[-1] - def _pyro_model(self, train_x: torch.Tensor, train_y: torch.Tensor) -> None: - # with gpytorch.settings.fast_computations(False, False, False): - sampled_model = self.pyro_sample_from_prior() + x_expanded = self.x.expand(X.shape[:-2] + (self.x.shape[-2], X.shape[-1])) - ys = sampled_model.likelihood(sampled_model.forward(train_x)) + cov_X_x = self.kernel_func(X, x_expanded) + cov_X_diff = cov_X_x[..., self.preferences[:, 0]] - cov_X_x[..., self.preferences[:, 1]] - pyro.sample("y", ys, obs=train_y) + mean = cov_X_diff @ (self._cov_diff_diff_inv @ self.diff) + cov = self.kernel_func(X, X) - cov_X_diff @ self._cov_diff_diff_inv @ cov_X_diff.transpose( + -1, -2 + ) + if observation_noise: + idx = torch.arange(cov.shape[-1]) + cov[..., idx, idx] += self.noise_var - def fit_mcmc( - self, X: torch.Tensor, preferences: torch.Tensor, cycles: int, rng: np.random.RandomState - ) -> None: + return botorch.posteriors.gpytorch.GPyTorchPosterior( + distribution=gpytorch.distributions.MultivariateNormal( + mean=mean, + covariance_matrix=cov, + ) + ) + + @property + def batch_shape(self) -> torch.Size: + return torch.Size() + + @property + def num_outputs(self) -> int: + return 1 + + +def _truncnorm_mean_var_logz(alpha: Tensor) -> tuple[Tensor, Tensor, Tensor]: + SQRT_HALF = math.sqrt(0.5) + SQRT_HALF_PI = math.sqrt(0.5 * math.pi) + logz = torch.special.log_ndtr(-alpha) + mean = 1 / (SQRT_HALF_PI * torch.special.erfcx(alpha * SQRT_HALF)) + var = 1 - mean * (mean - alpha) + return mean, var, logz + + +def _orthants_MVN_EP( + cov0: Tensor, preferences: Tensor, noise_var: Tensor, cycles: int +) -> tuple[Tensor, Tensor, Tensor]: + N = cov0.shape[0] + M = preferences.shape[0] + mu = torch.zeros(N, dtype=cov0.dtype) + cov = cov0.clone() + virtual_obs_a = [torch.tensor(0.0, dtype=cov0.dtype) for _ in range(M)] + virtual_obs_b = [torch.tensor(0.0, dtype=cov0.dtype) for _ in range(M)] + log_zs = torch.zeros(M, dtype=cov0.dtype) + + for _ in range(cycles): + for i in range(M): + pref_i = preferences[i, :] + mean1 = mu[pref_i[0]] - mu[pref_i[1]] + Sxy = cov[pref_i[0]] - cov[pref_i[1]] + var1 = Sxy[pref_i[0]] - Sxy[pref_i[1]] + + r0 = (1 - var1 * virtual_obs_a[i]).reciprocal() + var0 = var1 * r0 + mean0 = (mean1 + var1 * virtual_obs_b[i]) * r0 + + obs_var = var0 + noise_var + obs_sigma = torch.sqrt(obs_var) + alpha = -mean0 / torch.clamp_min(obs_sigma, min=1e-20) + mean_norm, var_norm, logz = _truncnorm_mean_var_logz(alpha) + + kalman_factor = var0 / torch.clamp_min(obs_var, min=1e-20) + mean2 = mean0 + obs_sigma * mean_norm * kalman_factor + var2 = kalman_factor * (noise_var + var_norm * var0) + + var1_var2_inv = torch.clamp_min(var1 * var2, min=1e-20).reciprocal() + db = (mean1 * var2 - mean2 * var1) * var1_var2_inv + da = (var1 - var2) * var1_var2_inv + virtual_obs_b[i] = virtual_obs_b[i] + db + virtual_obs_a[i] = virtual_obs_a[i] + da + + dr = (1 + var1 * da).reciprocal() + mu = mu - Sxy * ((db + mean1 * da) * dr) + cov = cov - (Sxy[:, None] * (da * dr)) @ Sxy[None, :] + log_zs[i] = logz + return mu, cov, torch.sum(log_zs) + + +_orthants_MVN_EP_jit = torch.jit.script(_orthants_MVN_EP) + + +class _PreferentialGP: + def __init__(self, kernel: gpytorch.kernels.Kernel, noise_prior: Prior, dims: int) -> None: + self.kernel = kernel + self.noise_prior = noise_prior + self.dims = dims + + self.diff = torch.empty((0,), dtype=torch.float64, requires_grad=False) + self.log_noise = torch.nn.Parameter( + torch.tensor(0.0, dtype=torch.float64), requires_grad=True + ) + + def fit_params_EP(self, X: Tensor, preferences: Tensor) -> None: if len(preferences) == 0: - # Skip actual MCMC computation - self.set_train_data( - inputs=torch.empty((0, X.shape[-1])), - targets=torch.empty((0,)), - strict=False, - ) - self.likelihood.weights = torch.empty((0,)) - else: - dtype = torch.float64 + return + tolerance = 1e-3 + max_iter = 100 - cnt = torch.bincount(preferences.reshape(-1)) - mask = cnt > 0 - train_x = X[mask] - weights = cnt[mask] + optim = torch.optim.LBFGS([*self.kernel.parameters(), self.log_noise]) - assert isinstance(self.likelihood, _WeightedGaussianLikelihood) - self.likelihood.weights = weights + last_params = [p.detach().clone() for p in optim.param_groups[0]["params"]] + for _ in range(max_iter): - preferences_np = preferences.detach().numpy() + def closure() -> Tensor: + optim.zero_grad() + noise = self.log_noise.exp() + cov0 = self.kernel.forward(X, X).to_dense() + _, _, logz = _orthants_MVN_EP_jit(cov0, preferences, noise, cycles=2) - all_ys_np = np.zeros((len(preferences), 2)) - train_y = torch.zeros( - ( - len( - train_x, - ) - ), - dtype=dtype, + loss = -logz - self.noise_prior.log_prob(noise) + for _, _, prior, param, _ in self.kernel.named_priors(): + loss = loss - prior.log_prob(param(self.kernel)).sum() + + loss.backward() + return loss + + optim.step(closure) + + # Check for convergence + params = optim.param_groups[0]["params"] + for p_old, p_new in zip(last_params, params): + if torch.max(torch.abs(p_old - p_new)) > tolerance: + break + else: + break + last_params = [p.detach().clone() for p in params] + + def sample_gp(self, x: Tensor, preferences: Tensor) -> _SampledGP: + self.fit_params_EP(x, preferences) + + with torch.no_grad(): + cov_diff_diff_inv = _compute_cov_diff_diff_inv( + preferences=preferences, + cov_x_x=self.kernel(x, x).to_dense(), + noise_var=self.log_noise.exp(), ) - nuts = pyro.infer.mcmc.NUTS( - model=self._pyro_model, - init_strategy=pyro.infer.autoguide.init_to_sample, - step_size=self._last_mcmc_step_size or 1.0, + original_diff_size = len(self.diff) + self.diff.resize_(len(preferences)) + self.diff[original_diff_size:] = 0.0 + + self.diff = _orthants_MVN_Gibbs_sampling_jit( + cov_inv=cov_diff_diff_inv, + initial_sample=self.diff, + cycles=20, + )[-1] + return _SampledGP( + kernel_func=lambda x1, x2: self.kernel(x1, x2).to_dense(), + x=x, + preferences=preferences, + noise_var=self.log_noise.exp(), + diff=self.diff, ) - warmup_steps = max(0, cycles - 2) - nuts.setup(warmup_steps=warmup_steps, train_x=train_x, train_y=train_y) - - raw_params = self._last_params or nuts.initial_params - for i in range(cycles): - params = { - name: nuts.transforms[name].inv(value) for name, value in raw_params.items() - } - _set_params(self, params) - self.set_train_data(train_x, train_y, strict=False) - all_ys_np = _sample_y( - preferences=preferences_np, - cov_X_X=self.covar_module(train_x).detach().numpy(), - obs_noise_var=float(self.likelihood.noise_covar.noise), - cycles=10, - initial_sample=all_ys_np, - rng=rng, - ) - ys_sum_np = np.zeros((len(X),)) - np.add.at(ys_sum_np, preferences_np.reshape(-1), all_ys_np.reshape(-1)) - ys_sum = torch.from_numpy(ys_sum_np) - train_y[:] = ys_sum[mask] / cnt[mask] - nuts.clear_cache() - try: - raw_params = nuts.sample(raw_params) - except NotPSDError: - nuts.cleanup() - nuts = pyro.infer.mcmc.NUTS( - model=self._pyro_model, - init_strategy=pyro.infer.autoguide.init_to_sample, - step_size=self._last_mcmc_step_size or 1.0, - ) - nuts.setup(warmup_steps=warmup_steps, train_x=train_x, train_y=train_y) - raw_params = nuts.initial_params - - params = {name: nuts.transforms[name].inv(value) for name, value in raw_params.items()} - self.set_train_data(train_x, train_y, strict=False) - _set_params(self, params) - - self._last_params = raw_params - self._last_mcmc_step_size = nuts.step_size - nuts.cleanup() - - def forward(self, x: torch.Tensor) -> gpytorch.distributions.MultivariateNormal: - mean_module = gpytorch.means.ZeroMean() - return gpytorch.distributions.MultivariateNormal( - mean_module(x), - self.covar_module(x), - ) - - -def _set_params( - module: gpytorch.Module, - params_dict: dict[str, torch.Tensor], - memo: set | None = None, - prefix: str = "", -) -> None: - if memo is None: - memo = set() - if hasattr(module, "_priors"): - for name, (prior, closure, setting_closure) in module._priors.items(): - if prior is not None and prior not in memo: - memo.add(prior) - setting_closure(module, params_dict[prefix + ("." if prefix else "") + name]) - - for mname, module_ in module.named_children(): - submodule_prefix = prefix + ("." if prefix else "") + mname - _set_params(module_, params_dict, memo=memo, prefix=submodule_prefix) class PreferentialGPSampler(optuna.samplers.BaseSampler): @@ -306,18 +282,16 @@ class PreferentialGPSampler(optuna.samplers.BaseSampler): noise_prior: Prior | None = None, independent_sampler: optuna.samplers.BaseSampler | None = None, seed: int | None = None, - device: torch.device | None = None, ) -> None: - self._rng = np.random.RandomState(seed) - self._search_space = IntersectionSearchSpace() - self.kernel = kernel - self.noise_prior = noise_prior - self.independent_sampler = independent_sampler or optuna.samplers.RandomSampler( - seed=self._rng.randint(2**32), - ) - self.device = device or torch.device("cpu") + self.noise_prior = noise_prior or gpytorch.priors.GammaPrior(5.0, 50.0) + self._rng = np.random.RandomState(seed) + self.independent_sampler = independent_sampler or optuna.samplers.RandomSampler( + seed=self._rng.randint(2**32) + ) + + self._search_space = optuna.search_space.IntersectionSearchSpace() self._gp: _PreferentialGP | None = None def reseed_rng(self) -> None: @@ -325,75 +299,64 @@ class PreferentialGPSampler(optuna.samplers.BaseSampler): self._rng = np.random.RandomState() def infer_relative_search_space( - self, study: Study, trial: FrozenTrial - ) -> dict[str, BaseDistribution]: + self, study: optuna.Study, trial: optuna.trial.FrozenTrial + ) -> dict[str, optuna.distributions.BaseDistribution]: return self._search_space.calculate(study) def sample_relative( self, - study: Study, - trial: FrozenTrial, - search_space: dict[str, BaseDistribution], + study: optuna.Study, + trial: optuna.trial.FrozenTrial, + search_space: dict[str, optuna.distributions.BaseDistribution], ) -> dict[str, Any]: + preferences = get_preferences(study.system_attrs) + if len(preferences) == 0: + return {} + + trials = study.get_trials(deepcopy=False) + trials_with_preference = list({t for (b, w) in preferences for t in (b, w)}) + ids = {t: i for i, t in enumerate(trials_with_preference)} + + trans = optuna._transform._SearchSpaceTransform( + search_space, transform_log=True, transform_step=True, transform_0_1=True + ) + params = torch.tensor( + np.array([trans.transform(trials[t].params) for t in trials_with_preference]), + dtype=torch.float64, + ) + pref_ids = torch.tensor([[ids[b], ids[w]] for b, w in preferences], dtype=torch.int32) with torch.random.fork_rng(): torch.manual_seed(self._rng.randint(2**32)) - pyro.set_rng_seed(self._rng.randint(2**32)) - if len(search_space) == 0: - return {} - - preferences = get_preferences(study._study_id, study._storage) - trials = study.get_trials(deepcopy=False) - if len(preferences) == 0: - return {} - - trans = _SearchSpaceTransform( - search_space, transform_log=True, transform_step=True, transform_0_1=True - ) - dims = len(trans.bounds) self._gp = self._gp or _PreferentialGP( kernel=self.kernel or gpytorch.kernels.MaternKernel( - nu=2.5, - ard_num_dims=dims, - lengthscale_prior=gpytorch.priors.GammaPrior(3.0, 6.0), - lengthscale_constraint=gpytorch.constraints.Positive(), + nu=1.5, + ard_num_dims=len(trans.bounds), + lengthscale_prior=gpytorch.priors.GammaPrior(5.0, 10.0), + lengthscale_constraint=gpytorch.constraints.GreaterThan( + 0.0, + transform=torch.exp, + inv_transform=torch.log, + ), ), - noise_prior=self.noise_prior or gpytorch.priors.GammaPrior(1.1, 2.0), - noise_constraint=gpytorch.constraints.Positive(), + noise_prior=self.noise_prior, + dims=len(trans.bounds), ) + if self._gp.dims != len(trans.bounds): + raise NotImplementedError( + "The search space has changed. " + "Dynamic search space is not supported in PreferentialGPSampler." + ) - ids: dict[int, int] = {} - params: list[torch.Tensor] = [] - pref_ids: list[tuple[int, int]] = [] - - for better, worse in preferences: - for t in (better, worse): - if t not in ids: - ids[t] = len(ids) - params.append(trans.transform(trials[t].params)) - pref_ids.append((ids[better], ids[worse])) - dtype = torch.float64 - - params_torch = torch.tensor(np.array(params), dtype=dtype, device=self.device) - pref_ids_torch = torch.tensor( - np.array(pref_ids), - dtype=torch.int32, - device=self.device, - ) - self._gp.fit_mcmc(params_torch, pref_ids_torch, cycles=10, rng=self._rng) - self._gp.eval() - scores = self._gp(params_torch).mean - - best_f = torch.max(scores) - - acqf = LogExpectedImprovement( - model=self._gp, - best_f=best_f, + sampled_gp = self._gp.sample_gp(params, pref_ids) + acqf = botorch.acquisition.analytic.LogExpectedImprovement( + model=sampled_gp, + best_f=torch.max(sampled_gp.posterior(params[:, None, :]).mean), ) # TODO: Make it possible to apply it on categorical variables - candidates, _ = optimize_acqf( + candidates, _ = botorch.optim.optimize_acqf( acq_function=acqf, bounds=torch.from_numpy(trans.bounds.T), q=1, @@ -407,10 +370,10 @@ class PreferentialGPSampler(optuna.samplers.BaseSampler): def sample_independent( self, - study: Study, - trial: FrozenTrial, + study: optuna.Study, + trial: optuna.trial.FrozenTrial, param_name: str, - param_distribution: distributions.BaseDistribution, + param_distribution: optuna.distributions.BaseDistribution, ) -> Any: return self.independent_sampler.sample_independent( study, trial, param_name, param_distribution diff --git a/optuna_dashboard/ts/action.ts b/optuna_dashboard/ts/action.ts index f0c82e2b..8979d3e3 100644 --- a/optuna_dashboard/ts/action.ts +++ b/optuna_dashboard/ts/action.ts @@ -588,10 +588,10 @@ export const actionCreator = () => { const updatePreference = ( study_id: number, - best_trials: number[], - worst_trials: number[] + candidates: number[], + clicked: number ) => { - reportPreferenceAPI(study_id, best_trials, worst_trials).catch((err) => { + reportPreferenceAPI(study_id, candidates, clicked).catch((err) => { const reason = err.response?.data.reason enqueueSnackbar(`Failed to report preference. Reason: ${reason}`, { variant: "error", diff --git a/optuna_dashboard/ts/apiClient.ts b/optuna_dashboard/ts/apiClient.ts index caa57716..6ae34403 100644 --- a/optuna_dashboard/ts/apiClient.ts +++ b/optuna_dashboard/ts/apiClient.ts @@ -55,6 +55,28 @@ const convertTrialResponse = (res: TrialResponse): Trial => { } } +interface PreferenceHistoryResponce { + id: string + preference_id: string + candidates: number[] + clicked: number + mode: PreferenceFeedbackMode + timestamp: string +} + +const convertPreferenceHistory = ( + res: PreferenceHistoryResponce +): PreferenceHistory => { + return { + id: res.id, + preference_id: res.preference_id, + candidates: res.candidates, + clicked: res.clicked, + feedback_mode: res.mode, + timestamp: new Date(res.timestamp), + } +} + interface StudyDetailResponse { name: string datetime_start: string @@ -70,8 +92,8 @@ interface StudyDetailResponse { is_preferential: boolean objective_names?: string[] form_widgets?: FormWidgets - feedback_component_type?: string - feedback_artifact_key?: string + preference_history?: PreferenceHistoryResponce[] + plotly_graph_objects: PlotlyGraphObject[] } export const getStudyDetailAPI = ( @@ -110,6 +132,10 @@ export const getStudyDetailAPI = ( feedback_component_type: res.data .feedback_component_type as FeedbackComponentType, feedback_artifact_key: res.data.feedback_artifact_key, + preference_history: res.data.preference_history?.map( + convertPreferenceHistory + ), + plotly_graph_objects: res.data.plotly_graph_objects, } }) } @@ -319,13 +345,14 @@ export const getParamImportances = ( export const reportPreferenceAPI = ( studyId: number, - best_trials: number[], - worst_trials: number[] + candidates: number[], + clicked: number ): Promise => { return axiosInstance .post(`/api/studies/${studyId}/preference`, { - best_trials: best_trials, - worst_trials: worst_trials, + candidates: candidates, + clicked: clicked, + mode: "ChooseWorst", }) .then(() => { return diff --git a/optuna_dashboard/ts/components/App.tsx b/optuna_dashboard/ts/components/App.tsx index 78015049..8adf8895 100644 --- a/optuna_dashboard/ts/components/App.tsx +++ b/optuna_dashboard/ts/components/App.tsx @@ -96,6 +96,15 @@ export const App: FC = () => { /> } /> + + } + /> } diff --git a/optuna_dashboard/ts/components/AppDrawer.tsx b/optuna_dashboard/ts/components/AppDrawer.tsx index 0903d72b..446b362e 100644 --- a/optuna_dashboard/ts/components/AppDrawer.tsx +++ b/optuna_dashboard/ts/components/AppDrawer.tsx @@ -34,12 +34,19 @@ import GitHubIcon from "@mui/icons-material/GitHub" import OpenInNewIcon from "@mui/icons-material/OpenInNew" import QueryStatsIcon from "@mui/icons-material/QueryStats" import ThumbUpAltIcon from "@mui/icons-material/ThumbUpAlt" +import HistoryIcon from "@mui/icons-material/History" import { Switch } from "@mui/material" import { actionCreator } from "../action" const drawerWidth = 240 -export type PageId = "top" | "analytics" | "trialTable" | "trialList" | "note" +export type PageId = + | "top" + | "analytics" + | "trialTable" + | "trialList" + | "note" + | "preferenceHistory" const openedMixin = (theme: Theme): CSSObject => ({ width: drawerWidth, @@ -204,6 +211,28 @@ export const AppDrawer: FC<{ /> + {isPreferential && ( + + + + + + + + + )} = ({ artifact, urlPath, height }) => { + if (isThreejsArtifact(artifact)) { + return ( + + ) + } else if (artifact.mimetype.startsWith("audio")) { + return ( + + ) + } else if (artifact.mimetype.startsWith("image")) { + return ( + + ) + } + return +} diff --git a/optuna_dashboard/ts/components/PreferenceHistory.tsx b/optuna_dashboard/ts/components/PreferenceHistory.tsx new file mode 100644 index 00000000..3bc5a750 --- /dev/null +++ b/optuna_dashboard/ts/components/PreferenceHistory.tsx @@ -0,0 +1,218 @@ +import React, { FC, useState } from "react" +import { + Typography, + Box, + useTheme, + Card, + CardContent, + CardActions, +} from "@mui/material" +import ClearIcon from "@mui/icons-material/Clear" +import IconButton from "@mui/material/IconButton" +import OpenInFullIcon from "@mui/icons-material/OpenInFull" +import Modal from "@mui/material/Modal" +import { red } from "@mui/material/colors" + +import { TrialListDetail } from "./TrialList" +import { MarkdownRenderer } from "./Note" +import { formatDate } from "../dateUtil" + +type TrialType = "worst" | "none" + +const CandidateTrial: FC<{ + trial: Trial + type: TrialType +}> = ({ trial, type }) => { + const theme = useTheme() + const trialWidth = 300 + const trialHeight = 300 + const [detailShown, setDetailShown] = useState(false) + + const cardComponentSx = { + padding: 0, + position: "relative", + overflow: "hidden", + "::before": {}, + } + if (type !== "none") { + cardComponentSx["::before"] = { + content: '""', + position: "absolute", + top: 0, + left: 0, + width: "100%", + height: "100%", + backgroundColor: theme.palette.mode === "dark" ? "white" : "black", + opacity: 0.2, + zIndex: 1, + transition: "opacity 0.3s ease-out", + } + } + + return ( + + + Trial {trial.number} + setDetailShown(true)} + aria-label="show detail" + > + + + + + + + + + {type === "worst" ? ( + + ) : null} + + setDetailShown(false)}> + + + false} + directions={[]} + objectiveNames={[]} + /> + + + + + ) +} + +const ChoiceTrials: FC<{ choice: PreferenceHistory; trials: Trial[] }> = ({ + choice, + trials, +}) => { + const theme = useTheme() + const worst_trials = new Set([choice.clicked]) + + return ( + + + {formatDate(choice.timestamp)} + + + {choice.candidates.map((trial_num, index) => ( + + ))} + + + ) +} + +export const PreferenceHistory: FC<{ studyDetail: StudyDetail | null }> = ({ + studyDetail, +}) => { + if ( + studyDetail === null || + !studyDetail.is_preferential || + studyDetail.preference_history === undefined + ) { + return null + } + const theme = useTheme() + const preference_histories = [...studyDetail.preference_history] + + if (preference_histories.length === 0) { + return ( + + No feedback history + + ) + } + + return ( + + {preference_histories.reverse().map((choice) => ( + + ))} + + ) +} diff --git a/optuna_dashboard/ts/components/PreferentialTrials.tsx b/optuna_dashboard/ts/components/PreferentialTrials.tsx index ec597a09..e68e7ec9 100644 --- a/optuna_dashboard/ts/components/PreferentialTrials.tsx +++ b/optuna_dashboard/ts/components/PreferentialTrials.tsx @@ -13,46 +13,23 @@ import { FormLabel, Modal, } from "@mui/material" +import IconButton from "@mui/material/IconButton" import OpenInFullIcon from "@mui/icons-material/OpenInFull" import ReplayIcon from "@mui/icons-material/Replay" import ClearIcon from "@mui/icons-material/Clear" -import IconButton from "@mui/material/IconButton" import SettingsIcon from "@mui/icons-material/Settings" +import FullscreenIcon from "@mui/icons-material/Fullscreen" import red from "@mui/material/colors/red" + import { actionCreator } from "../action" -import { MarkdownRenderer } from "./Note" +import { TrialListDetail } from "./TrialList" import { - TrialArtifactActions, - TrialArtifactContent, - TrialListDetail, -} from "./TrialList" - -const FeedbackContent: FC<{ - trial: Trial - artifact?: Artifact - componentId: FeedbackComponentType - width: string - minHeight: string -}> = ({ trial, artifact, componentId, width, minHeight }) => { - if (componentId === "Note") { - return - } - if (componentId === "Artifact") { - if (artifact === undefined) { - return null - } - return ( - - ) - } - - return null -} + isThreejsArtifact, + useThreejsArtifactModal, +} from "./ThreejsArtifactViewer" +import { ArtifactCardMedia } from "./ArtifactCardMedia" +import { MarkdownRenderer } from "./Note" +import { Details } from "@mui/icons-material" const ModalPage: FC<{ children: React.ReactNode @@ -73,7 +50,7 @@ const ModalPage: FC<{ maxHeight: "90%", margin: "auto", overflow: "hidden", - backgroundColor: theme.palette.mode === "dark" ? "black" : "white", + backgroundColor: theme.palette.background.default, borderRadius: theme.spacing(3), }} > @@ -91,177 +68,6 @@ const ModalPage: FC<{ ) } -const PreferentialTrial: FC<{ - trial?: Trial - studyDetail: StudyDetail - hideTrial: () => void -}> = ({ trial, studyDetail, hideTrial }) => { - const theme = useTheme() - const action = actionCreator() - const trialWidth = 400 - const trialHeight = 300 - const [detailShown, setDetailShown] = useState(false) - const [buttonHover, setButtonHover] = useState(false) - const componentId = studyDetail.feedback_component_type ?? "Note" - const artifactKey = studyDetail.feedback_artifact_key - const artifactId = trial?.user_attrs.find((a) => a.key === artifactKey)?.value - const artifact = trial?.artifacts.find((a) => a.artifact_id === artifactId) - - if (trial == undefined) { - return ( - - ) - } - const onFeedback = () => { - hideTrial() - const best_trials = studyDetail.best_trials - .map((t) => t.number) - .filter((t) => t !== trial.number) - action.updatePreference(trial.study_id, best_trials, [trial.number]) - } - - return ( - - - Trial {trial.number} - {componentId === "Artifact" && artifact !== undefined ? ( - - {`(${artifact.filename})`} - - ) : null} - {componentId === "Artifact" && artifact !== undefined ? ( - - ) : null} - { - hideTrial() - action.skipPreferentialTrial(trial.study_id, trial.trial_id) - }} - aria-label="skip trial" - > - - - setDetailShown(true)} - aria-label="show detail" - > - - - - { - if (e.shiftKey) onFeedback() - }} - sx={{ - position: "relative", - padding: theme.spacing(2), - overflow: "hidden", - minHeight: theme.spacing(20), - }} - > - - - - - - - - { - setDetailShown(false) - }} - > - true} - directions={[]} - objectiveNames={[]} - /> - - - ) -} - const SettingsPage: FC<{ studyDetail: StudyDetail settingShown: boolean @@ -358,6 +164,29 @@ const SettingsPage: FC<{ ) } +const FeedbackContent: FC<{ + trial: Trial + artifact?: Artifact + componentId: FeedbackComponentType + width: string + minHeight: string + urlPath: string +}> = ({ trial, artifact, componentId, width, minHeight, urlPath }) => { + if (componentId === "Note") { + return + } + if (componentId === "Artifact") { + if (artifact === undefined) { + return null + } + return ( + + ) + } + + return null +} + type DisplayTrials = { numbers: number[] last_number: number @@ -366,16 +195,29 @@ type DisplayTrials = { export const PreferentialTrials: FC<{ studyDetail: StudyDetail | null }> = ({ studyDetail, }) => { + const theme = useTheme() + const action = actionCreator() + const [openThreejsArtifactModal, renderThreejsArtifactModal] = + useThreejsArtifactModal() + const runningTrials = + studyDetail?.trials.filter((t) => t.state === "Running") ?? [] + const activeTrials = runningTrials.concat(studyDetail?.best_trials ?? []) + + const [displayTrials, setDisplayTrials] = useState({ + numbers: activeTrials.map((t) => t.number), + last_number: Math.max(...activeTrials.map((t) => t.number), -1), + }) + const [settingShown, setSettingShown] = useState(false) + const [detailTrial, setDetailTrial] = useState(null) + const [buttonHover, setButtonHover] = useState(null) + + const trialWidth = 400 + const trialHeight = 300 + if (studyDetail === null || !studyDetail.is_preferential) { return null } - const theme = useTheme() - const [displayTrials, setDisplayTrials] = useState({ - numbers: studyDetail.best_trials.map((t) => t.number), - last_number: Math.max(...studyDetail.best_trials.map((t) => t.number), -1), - }) - const [settingShown, setSettingShown] = useState(false) - const new_trails = studyDetail.best_trials.filter( + const new_trails = activeTrials.filter( (t) => displayTrials.last_number < t.number && displayTrials.numbers.find((n) => n === t.number) === undefined @@ -441,22 +283,209 @@ export const PreferentialTrials: FC<{ studyDetail: StudyDetail | null }> = ({ Which trial is the worst? - {displayTrials.numbers.map((t, index) => ( - trial.number === t)} - studyDetail={studyDetail} - hideTrial={() => { - hideTrial(t) - }} - /> - ))} + {displayTrials.numbers.map((t, index) => { + const trial = activeTrials.find((trial) => trial.number === t) + const candidates = displayTrials.numbers.filter((n) => n !== -1) + const componentId = studyDetail.feedback_component_type ?? "Note" + const artifactKey = studyDetail.feedback_artifact_key + const artifactId = trial?.user_attrs.find( + (a) => a.key === artifactKey + )?.value + const artifact = trial?.artifacts.find( + (a) => a.artifact_id === artifactId + ) + const urlPath = `/artifacts/${studyDetail.id}/${trial?.trial_id}/${artifact?.artifact_id}` + + if (trial == undefined) { + return ( + + ) + } + + const is3dModel = + componentId === "Artifact" && + artifact !== undefined && + isThreejsArtifact(artifact) + const onFeedback = () => { + hideTrial(trial.number) + action.updatePreference(trial.study_id, candidates, trial.number) + } + + return ( + + + + Trial {trial.number} + {componentId === "Artifact" && artifact !== undefined ? ( + + {`(${artifact.filename})`} + + ) : null} + + {is3dModel ? ( + { + openThreejsArtifactModal(urlPath, artifact) + }} + > + + + ) : null} + { + hideTrial(trial.number) + action.skipPreferentialTrial(trial.study_id, trial.trial_id) + }} + aria-label="skip trial" + > + + + setDetailTrial(trial.number)} + aria-label="show detail" + > + + + + { + if (e.shiftKey) onFeedback() + }} + sx={{ + position: "relative", + padding: theme.spacing(2), + overflow: "hidden", + minHeight: theme.spacing(20), + }} + > + + + + + + + ) + })} + {detailTrial !== null && ( + { + setDetailTrial(null) + }} + > + + studyDetail.trials.find((t) => t.trial_id === trialId)?.state === + "Complete" ?? false + } + directions={[]} + objectiveNames={[]} + /> + + )} + {renderThreejsArtifactModal()} ) } diff --git a/optuna_dashboard/ts/components/StudyDetail.tsx b/optuna_dashboard/ts/components/StudyDetail.tsx index fedfd625..15cb7cf8 100644 --- a/optuna_dashboard/ts/components/StudyDetail.tsx +++ b/optuna_dashboard/ts/components/StudyDetail.tsx @@ -30,6 +30,7 @@ import { GraphEdf } from "./GraphEdf" import { TrialList } from "./TrialList" import { StudyHistory } from "./StudyHistory" import { PreferentialTrials } from "./PreferentialTrials" +import { PreferenceHistory } from "./PreferenceHistory" import { PreferentialAnalytics } from "./PreferentialAnalytics" interface ParamTypes { @@ -175,6 +176,8 @@ export const StudyDetail: FC<{ /> ) + } else if (page == "preferenceHistory") { + content = } const toolbar = ( diff --git a/optuna_dashboard/ts/components/StudyHistory.tsx b/optuna_dashboard/ts/components/StudyHistory.tsx index b47c557a..907acd1a 100644 --- a/optuna_dashboard/ts/components/StudyHistory.tsx +++ b/optuna_dashboard/ts/components/StudyHistory.tsx @@ -15,6 +15,7 @@ import { GraphIntermediateValues } from "./GraphIntermediateValues" import Grid2 from "@mui/material/Unstable_Grid2" import { DataGrid, DataGridColumn } from "./DataGrid" import { GraphHyperparameterImportance } from "./GraphHyperparameterImportances" +import { UserDefinedPlot } from "./UserDefinedPlot" import { BestTrialsCard } from "./BestTrialsCard" import { useStudyDetailValue, @@ -124,6 +125,16 @@ export const StudyHistory: FC<{ studyId: number }> = ({ studyId }) => { + {studyDetail !== null && + studyDetail.plotly_graph_objects.map((go) => ( + + + + + + + + ))} diff --git a/optuna_dashboard/ts/components/ThreejsArtifactViewer.tsx b/optuna_dashboard/ts/components/ThreejsArtifactViewer.tsx index 536f1d91..a88b9d2c 100644 --- a/optuna_dashboard/ts/components/ThreejsArtifactViewer.tsx +++ b/optuna_dashboard/ts/components/ThreejsArtifactViewer.tsx @@ -1,10 +1,17 @@ import * as THREE from "three" -import React, { useEffect, useState } from "react" +import React, { useEffect, useState, ReactNode } from "react" import { Canvas } from "@react-three/fiber" import { GizmoHelper, GizmoViewport, OrbitControls } from "@react-three/drei" import { STLLoader } from "three/examples/jsm/loaders/STLLoader" import { Rhino3dmLoader } from "three/examples/jsm/loaders/3DMLoader" import { PerspectiveCamera } from "three" +import { Modal, Box } from "@mui/material" + +export const isThreejsArtifact = (artifact: Artifact): boolean => { + return ( + artifact.filename.endsWith(".stl") || artifact.filename.endsWith(".3dm") + ) +} interface ThreejsArtifactViewerProps { src: string @@ -109,3 +116,48 @@ export const ThreejsArtifactViewer: React.FC = ( ) } + +export const useThreejsArtifactModal = (): [ + (path: string, artifact: Artifact) => void, + () => ReactNode +] => { + const [open, setOpen] = useState(false) + const [target, setTarget] = useState<[string, Artifact | null]>(["", null]) + + const openModal = (artifactUrlPath: string, artifact: Artifact) => { + setTarget([artifactUrlPath, artifact]) + setOpen(true) + } + + const renderDeleteStudyDialog = () => { + return ( + { + setOpen(false) + setTarget(["", null]) + }} + > + + + + + ) + } + return [openModal, renderDeleteStudyDialog] +} diff --git a/optuna_dashboard/ts/components/TrialArtifactCards.tsx b/optuna_dashboard/ts/components/TrialArtifactCards.tsx new file mode 100644 index 00000000..3e15f7ed --- /dev/null +++ b/optuna_dashboard/ts/components/TrialArtifactCards.tsx @@ -0,0 +1,233 @@ +import React, { + ChangeEventHandler, + DragEventHandler, + FC, + MouseEventHandler, + useRef, + useState, +} from "react" +import { + Typography, + Box, + useTheme, + IconButton, + Card, + CardContent, + CardActionArea, +} from "@mui/material" +import UploadFileIcon from "@mui/icons-material/UploadFile" +import DownloadIcon from "@mui/icons-material/Download" +import DeleteIcon from "@mui/icons-material/Delete" +import FullscreenIcon from "@mui/icons-material/Fullscreen" + +import { actionCreator } from "../action" +import { useDeleteArtifactDialog } from "./DeleteArtifactDialog" +import { + useThreejsArtifactModal, + isThreejsArtifact, +} from "./ThreejsArtifactViewer" +import { ArtifactCardMedia } from "./ArtifactCardMedia" + +export const TrialArtifactCards: FC<{ trial: Trial }> = ({ trial }) => { + const theme = useTheme() + const [openDeleteArtifactDialog, renderDeleteArtifactDialog] = + useDeleteArtifactDialog() + const [openThreejsArtifactModal, renderThreejsArtifactModal] = + useThreejsArtifactModal() + + const width = "200px" + const height = "150px" + + return ( + <> + + Artifacts + + + {trial.artifacts.map((artifact) => { + const urlPath = `/artifacts/${trial.study_id}/${trial.trial_id}/${artifact.artifact_id}` + return ( + + + + + {artifact.filename} + + {isThreejsArtifact(artifact) ? ( + { + openThreejsArtifactModal(urlPath, artifact) + }} + > + + + ) : null} + { + openDeleteArtifactDialog( + trial.study_id, + trial.trial_id, + artifact + ) + }} + > + + + + + + + + ) + })} + + + {renderDeleteArtifactDialog()} + {renderThreejsArtifactModal()} + + ) +} + +const TrialArtifactUploader: FC<{ + trial: Trial + width: string + height: string +}> = ({ trial, width, height }) => { + const theme = useTheme() + const action = actionCreator() + const [dragOver, setDragOver] = useState(false) + + if (trial.state !== "Running" && trial.state !== "Waiting") { + return null + } + const inputRef = useRef(null) + const handleClick: MouseEventHandler = () => { + if (!inputRef || !inputRef.current) { + return + } + inputRef.current.click() + } + const handleOnChange: ChangeEventHandler = (e) => { + const files = e.target.files + if (files === null) { + return + } + action.uploadArtifact(trial.study_id, trial.trial_id, files[0]) + } + const handleDrop: DragEventHandler = (e) => { + e.stopPropagation() + e.preventDefault() + const files = e.dataTransfer.files + setDragOver(false) + for (let i = 0; i < files.length; i++) { + action.uploadArtifact(trial.study_id, trial.trial_id, files[i]) + } + } + const handleDragOver: DragEventHandler = (e) => { + e.stopPropagation() + e.preventDefault() + e.dataTransfer.dropEffect = "copy" + setDragOver(true) + } + const handleDragLeave: DragEventHandler = (e) => { + e.stopPropagation() + e.preventDefault() + e.dataTransfer.dropEffect = "copy" + setDragOver(false) + } + return ( + + + + + + Upload a New File + + Drag your file here or click to browse. + + + + + ) +} diff --git a/optuna_dashboard/ts/components/TrialList.tsx b/optuna_dashboard/ts/components/TrialList.tsx index bb70c119..6922003a 100644 --- a/optuna_dashboard/ts/components/TrialList.tsx +++ b/optuna_dashboard/ts/components/TrialList.tsx @@ -1,13 +1,4 @@ -import React, { - ChangeEventHandler, - DragEventHandler, - FC, - MouseEventHandler, - ReactNode, - useMemo, - useRef, - useState, -} from "react" +import React, { FC, ReactNode, useMemo } from "react" import { Typography, Box, @@ -16,13 +7,7 @@ import { IconButton, Menu, MenuItem, - Card, - CardContent, - CardMedia, - CardActionArea, - Modal, } from "@mui/material" -import { SxProps } from "@mui/system" import Chip from "@mui/material/Chip" import Divider from "@mui/material/Divider" import List from "@mui/material/List" @@ -33,11 +18,6 @@ import ListSubheader from "@mui/material/ListSubheader" import FilterListIcon from "@mui/icons-material/FilterList" import CheckBoxOutlineBlankIcon from "@mui/icons-material/CheckBoxOutlineBlank" import CheckBoxIcon from "@mui/icons-material/CheckBox" -import UploadFileIcon from "@mui/icons-material/UploadFile" -import DownloadIcon from "@mui/icons-material/Download" -import DeleteIcon from "@mui/icons-material/Delete" -import FullscreenIcon from "@mui/icons-material/Fullscreen" -import InsertDriveFileIcon from "@mui/icons-material/InsertDriveFile" import StopCircleIcon from "@mui/icons-material/StopCircle" import { TrialNote } from "./Note" @@ -46,9 +26,8 @@ import ListItemIcon from "@mui/material/ListItemIcon" import { useRecoilValue } from "recoil" import { artifactIsAvailable } from "../state" import { actionCreator } from "../action" -import { useDeleteArtifactDialog } from "./DeleteArtifactDialog" import { TrialFormWidgets } from "./TrialFormWidgets" -import { ThreejsArtifactViewer } from "./ThreejsArtifactViewer" +import { TrialArtifactCards } from "./TrialArtifactCards" const states: TrialState[] = [ "Complete", @@ -320,344 +299,11 @@ export const TrialListDetail: FC<{ value !== null ? renderInfo(key, value) : null )} - {artifactEnabled && } + {artifactEnabled && } ) } -export const TrialArtifactContent: FC<{ - trial: Trial - artifact: Artifact - width: string - height: string -}> = ({ trial, artifact, width, height }) => { - if (artifact.mimetype.startsWith("image")) { - return ( - - ) - } else if ( - artifact.filename.endsWith(".stl") || - artifact.filename.endsWith(".3dm") - ) { - return ( - - - - ) - } else if (artifact.mimetype.startsWith("audio")) { - return ( - - - - ) - } else { - return ( - - - - ) - } -} - -export const TrialArtifactActions: FC<{ - trial: Trial - artifact: Artifact - sx: SxProps -}> = ({ trial, artifact, sx }) => { - const [open3dModelViewer, setOpen3dModelViewer] = useState(false) - - if (artifact.mimetype.startsWith("image")) { - return null - } else if ( - artifact.filename.endsWith(".stl") || - artifact.filename.endsWith(".3dm") - ) { - return ( - <> - { - setOpen3dModelViewer(true) - }} - > - - - { - setOpen3dModelViewer(false) - }} - > - - - - - - ) - } - return null -} - -const TrialArtifact: FC<{ - trial: Trial - artifact: Artifact - width: string - height: string -}> = ({ trial, artifact, width, height }) => { - const [openDeleteArtifactDialog, renderDeleteArtifactDialog] = - useDeleteArtifactDialog() - const theme = useTheme() - const is3dModel = - artifact.filename.endsWith(".stl") || artifact.filename.endsWith(".3dm") - const canDelete = trial.state === "Running" || trial.state === "Waiting" - let actionsCount = 1 - if (canDelete) actionsCount += 1 - if (is3dModel) actionsCount += 1 - const actionsWidth = theme.spacing(actionsCount * 4) - - return ( - - - - - {artifact.filename} - - {is3dModel ? ( - - ) : null} - {canDelete ? ( - { - openDeleteArtifactDialog(trial.study_id, trial.trial_id, artifact) - }} - > - - - ) : null} - - - - - {renderDeleteArtifactDialog()} - - ) -} - -const TrialArtifacts: FC<{ trial: Trial }> = ({ trial }) => { - const theme = useTheme() - const action = actionCreator() - const [dragOver, setDragOver] = useState(false) - - const width = "200px" - const height = "150px" - - const inputRef = useRef(null) - const handleClick: MouseEventHandler = () => { - if (!inputRef || !inputRef.current) { - return - } - inputRef.current.click() - } - const handleOnChange: ChangeEventHandler = (e) => { - const files = e.target.files - if (files === null) { - return - } - action.uploadArtifact(trial.study_id, trial.trial_id, files[0]) - } - const handleDrop: DragEventHandler = (e) => { - e.stopPropagation() - e.preventDefault() - const files = e.dataTransfer.files - setDragOver(false) - for (let i = 0; i < files.length; i++) { - action.uploadArtifact(trial.study_id, trial.trial_id, files[i]) - } - } - const handleDragOver: DragEventHandler = (e) => { - e.stopPropagation() - e.preventDefault() - e.dataTransfer.dropEffect = "copy" - setDragOver(true) - } - const handleDragLeave: DragEventHandler = (e) => { - e.stopPropagation() - e.preventDefault() - e.dataTransfer.dropEffect = "copy" - setDragOver(false) - } - - return ( - <> - - Artifacts - - - {trial.artifacts.map((a) => ( - - ))} - {trial.state === "Running" || trial.state === "Waiting" ? ( - - - - - - Upload a New File - - Drag your file here or click to browse. - - - - - ) : null} - - - ) -} - const getTrialListLink = ( studyId: number, exclude: TrialState[], diff --git a/optuna_dashboard/ts/components/UserDefinedPlot.tsx b/optuna_dashboard/ts/components/UserDefinedPlot.tsx new file mode 100644 index 00000000..029c4b57 --- /dev/null +++ b/optuna_dashboard/ts/components/UserDefinedPlot.tsx @@ -0,0 +1,21 @@ +import * as plotly from "plotly.js-dist-min" +import React, { FC, useEffect } from "react" +import { Box } from "@mui/material" + +export const UserDefinedPlot: FC<{ + graphObject: PlotlyGraphObject +}> = ({ graphObject }) => { + const plotDomId = `user-defined-plot:${graphObject.id}` + + useEffect(() => { + try { + const parsed = JSON.parse(graphObject.graph_object) + plotly.react(plotDomId, parsed.data, parsed.layout) + } catch (e) { + // Avoid to crash the whole page when given invalid grpah objects. + console.error(e) + } + }, [graphObject]) + + return +} diff --git a/optuna_dashboard/ts/types/index.d.ts b/optuna_dashboard/ts/types/index.d.ts index 241ea310..66fc06ed 100644 --- a/optuna_dashboard/ts/types/index.d.ts +++ b/optuna_dashboard/ts/types/index.d.ts @@ -12,6 +12,7 @@ type TrialIntermediateValueNumber = number | "inf" | "-inf" | "nan" type TrialState = "Running" | "Complete" | "Pruned" | "Fail" | "Waiting" type TrialStateFinished = "Complete" | "Fail" | "Pruned" type StudyDirection = "maximize" | "minimize" | "not_set" +type PreferenceFeedbackMode = "ChooseWorst" type FeedbackComponentType = "Note" | "Artifact" type FloatDistribution = { @@ -182,6 +183,11 @@ type FormWidgets = widgets: UserAttrFormWidget[] } +type PlotlyGraphObject = { + id: string + graph_object: string +} + type StudyDetail = { id: number name: string @@ -200,6 +206,8 @@ type StudyDetail = { form_widgets?: FormWidgets feedback_component_type?: FeedbackComponentType feedback_artifact_key?: string + preference_history?: PreferenceHistory[] + plotly_graph_objects: PlotlyGraphObject[] } type StudyDetails = { @@ -209,3 +217,12 @@ type StudyDetails = { type StudyParamImportance = { [study_id: string]: ParamImportance[][] } + +type PreferenceHistory = { + id: string + preference_id: string + candidates: number[] + clicked: number + feedback_mode: PreferenceFeedbackMode + timestamp: Date +} diff --git a/pyproject.toml b/pyproject.toml index 0555f701..9b7ce731 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,6 +43,7 @@ docs = [ test = [ "coverage", + "plotly", "pytest", "moto[s3]", ] diff --git a/python_tests/preferential/test_study.py b/python_tests/preferential/test_study.py index 3de57dae..7fc1aaba 100644 --- a/python_tests/preferential/test_study.py +++ b/python_tests/preferential/test_study.py @@ -40,7 +40,6 @@ def test_report_and_get_preferences(storage_supplier: Callable[[], StorageSuppli for _ in range(2): trial = study.ask() trial.suggest_float("x", 0, 1) - study.mark_comparison_ready(trial) better, worse = study.trials study.report_preference(better, worse) assert len(study.preferences) == 1 @@ -152,7 +151,6 @@ def test_copy_study() -> None: for _ in range(3): trial = from_study.ask() trial.suggest_float("x", 0, 1) - from_study.mark_comparison_ready(trial) from_study.report_preference(from_study.trials[0], from_study.trials[1]) from_study.report_preference(from_study.trials[1], from_study.trials[2]) @@ -243,7 +241,6 @@ def test_get_trials(storage_supplier: Callable[[], StorageSupplier]) -> None: for _ in range(5): trial = study.ask() trial.suggest_int("x", 1, 5) - study.mark_comparison_ready(trial) with patch("copy.deepcopy", wraps=copy.deepcopy) as mock_object: trials0 = study.get_trials(deepcopy=False) @@ -266,8 +263,7 @@ def test_get_trials_state_option(storage_supplier: Callable[[], StorageSupplier] with storage_supplier() as storage: study = create_study(n_generate=4, storage=storage) for _ in range(3): - trial = study.ask() - study.mark_comparison_ready(trial) + study.ask() better, worse = study.trials[:2] study.report_preference(better, worse) diff --git a/python_tests/preferential/test_system_attrs.py b/python_tests/preferential/test_system_attrs.py index 10448d48..34f93200 100644 --- a/python_tests/preferential/test_system_attrs.py +++ b/python_tests/preferential/test_system_attrs.py @@ -18,12 +18,13 @@ def test_report_and_get_preferences(storage_supplier: Callable[[], StorageSuppli study.ask() study_id = study._study_id - assert len(get_preferences(study_id, storage)) == 0 + + assert len(get_preferences(storage.get_study_system_attrs(study_id))) == 0 better, worse = study.trials[0], study.trials[1] report_preferences(study_id, storage, [(better.number, worse.number)]) - assert len(get_preferences(study_id, storage)) == 1 + assert len(get_preferences(storage.get_study_system_attrs(study_id))) == 1 - actual_better, actual_worse = get_preferences(study_id, storage)[0] + actual_better, actual_worse = get_preferences(storage.get_study_system_attrs(study_id))[0] assert actual_better == better.number assert actual_worse == worse.number diff --git a/python_tests/test_api.py b/python_tests/test_api.py index 7633cd8f..29e4da9f 100644 --- a/python_tests/test_api.py +++ b/python_tests/test_api.py @@ -105,10 +105,11 @@ class APITestCase(TestCase): storage = optuna.storages.InMemoryStorage() study = create_study(n_generate=4, storage=storage) for _ in range(3): - trial = study.ask() - study.mark_comparison_ready(trial) + study.ask() study.report_preference(study.trials[0], study.trials[1]) + assert len(study.best_trials) == 1 + app = create_app(storage) study_id = study._study._study_id status, _, body = send_request( @@ -120,16 +121,14 @@ class APITestCase(TestCase): self.assertEqual(status, 200) best_trials = json.loads(body)["best_trials"] - assert len(best_trials) == 2 + assert len(best_trials) == 1 assert best_trials[0]["number"] == 0 - assert best_trials[1]["number"] == 2 def test_report_preference(self) -> None: storage = optuna.storages.InMemoryStorage() study = create_study(n_generate=4, storage=storage) for _ in range(3): - trial = study.ask() - study.mark_comparison_ready(trial) + study.ask() app = create_app(storage) study_id = study._study._study_id @@ -137,7 +136,13 @@ class APITestCase(TestCase): app, f"/api/studies/{study_id}/preference", "POST", - body=json.dumps({"best_trials": [0, 2], "worst_trials": [1]}), + body=json.dumps( + { + "mode": "ChooseWorst", + "candidates": [0, 1, 2], + "clicked": 1, + } + ), content_type="application/json", ) self.assertEqual(status, 204) @@ -152,13 +157,35 @@ class APITestCase(TestCase): assert better.number == 2 assert worse.number == 1 + def test_report_preference_when_typo_mode(self) -> None: + storage = optuna.storages.InMemoryStorage() + study = create_study(storage=storage, n_generate=3) + for _ in range(3): + study.ask() + + app = create_app(storage) + study_id = study._study._study_id + status, _, _ = send_request( + app, + f"/api/studies/{study_id}/preference", + "POST", + body=json.dumps( + { + "mode": "ChoseWorst", + "candidates": [0, 1, 2], + "clicked": 1, + } + ), + content_type="application/json", + ) + self.assertEqual(status, 400) + def test_change_component(self) -> None: storage = optuna.storages.InMemoryStorage() - study = create_study(storage=storage) + study = create_study(storage=storage, n_generate=3) register_output_component(study, "Note") for _ in range(3): - trial = study.ask() - study.mark_comparison_ready(trial) + study.ask() app = create_app(storage) study_id = study._study._study_id @@ -220,23 +247,23 @@ class APITestCase(TestCase): trials: list[optuna.Trial] = [] for _ in range(3): trial = study.ask() - study.mark_comparison_ready(trial) trials.append(trial) + study.report_preference(study.trials[0], study.trials[1]) + study.report_preference(study.trials[2], study.trials[1]) app = create_app(storage) study_id = study._study._study_id status, _, _ = send_request( app, - f"/api/studies/{study_id}/{trials[1]._trial_id}/skip", + f"/api/studies/{study_id}/{trials[0]._trial_id}/skip", "POST", content_type="application/json", ) self.assertEqual(status, 204) best_trials = study.best_trials - assert len(best_trials) == 2 - assert best_trials[0].number == 0 - assert best_trials[1].number == 2 + assert len(best_trials) == 1 + assert best_trials[0].number == 2 def test_create_study(self) -> None: for name, directions, expected_status in [ diff --git a/python_tests/test_custom_plot_data.py b/python_tests/test_custom_plot_data.py new file mode 100644 index 00000000..3dcfc856 --- /dev/null +++ b/python_tests/test_custom_plot_data.py @@ -0,0 +1,86 @@ +from __future__ import annotations + +import optuna +from optuna_dashboard import _custom_plot_data as custom_plot_data +from optuna_dashboard import save_plotly_graph_object +import pytest + + +def get_dummy_study() -> optuna.Study: + def objective(trial: optuna.Trial) -> float: + x = trial.suggest_float("x", -100, 100) + y = trial.suggest_categorical("y", [-1, 0, 1]) + return x**2 + y + + study = optuna.create_study() + optuna.logging.set_verbosity(optuna.logging.ERROR) + study.optimize(objective, n_trials=100) + return study + + +def test_save_plotly_graph_object() -> None: + # Save history plot + dummy_study = get_dummy_study() + plot_data = optuna.visualization.plot_optimization_history(dummy_study) + graph_object_id = save_plotly_graph_object(dummy_study, plot_data) + + study_system_attrs = dummy_study._storage.get_study_system_attrs(dummy_study._study_id) + plot_data_dict = custom_plot_data.get_plotly_graph_objects(study_system_attrs) + assert len(plot_data_dict) == 1 + assert plot_data_dict[graph_object_id] == plot_data.to_json() + + # Save parallel coordinate plot + plot_data = optuna.visualization.plot_parallel_coordinate(dummy_study) + graph_object_id = save_plotly_graph_object(dummy_study, plot_data) + + study_system_attrs = dummy_study._storage.get_study_system_attrs(dummy_study._study_id) + plot_data_dict = custom_plot_data.get_plotly_graph_objects(study_system_attrs) + assert len(plot_data_dict) == 2 + assert plot_data_dict[graph_object_id] == plot_data.to_json() + + +def test_update_plotly_graph_object() -> None: + # Save history plot + dummy_study = get_dummy_study() + plot_data = optuna.visualization.plot_optimization_history(dummy_study) + graph_object_id = save_plotly_graph_object(dummy_study, plot_data) + + study_system_attrs = dummy_study._storage.get_study_system_attrs(dummy_study._study_id) + plot_data_dict = custom_plot_data.get_plotly_graph_objects(study_system_attrs) + assert len(plot_data_dict) == 1 + assert plot_data_dict[graph_object_id] == plot_data.to_json() + + # Save parallel coordinate plot + plot_data = optuna.visualization.plot_parallel_coordinate(dummy_study) + graph_object_id = save_plotly_graph_object( + dummy_study, plot_data, graph_object_id=graph_object_id + ) + + study_system_attrs = dummy_study._storage.get_study_system_attrs(dummy_study._study_id) + plot_data_dict = custom_plot_data.get_plotly_graph_objects(study_system_attrs) + assert len(plot_data_dict) == 1 + assert plot_data_dict[graph_object_id] == plot_data.to_json() + + +@pytest.mark.parametrize( + "name", + [ + "0", + "a", + "a1-:_.", + ], +) +def test_is_valid_graph_object_id(name: str) -> None: + assert custom_plot_data.is_valid_graph_object_id(name) + + +@pytest.mark.parametrize( + "name", + [ + "a,", + "a b", + "aあいうえお", + ], +) +def test_is_invalid_graph_object_id(name: str) -> None: + assert not custom_plot_data.is_valid_graph_object_id(name) diff --git a/python_tests/test_preferential_history.py b/python_tests/test_preferential_history.py new file mode 100644 index 00000000..51c9b0f8 --- /dev/null +++ b/python_tests/test_preferential_history.py @@ -0,0 +1,61 @@ +from __future__ import annotations + +from typing import Callable + +from optuna_dashboard._preferential_history import NewHistory +from optuna_dashboard._preferential_history import report_history +from optuna_dashboard._serializer import serialize_preference_history +from optuna_dashboard.preferential import create_study +from optuna_dashboard.preferential._system_attrs import _SYSTEM_ATTR_PREFIX_PREFERENCE + +from .storage_supplier import parametrize_storages +from .storage_supplier import StorageSupplier + + +@parametrize_storages +def test_report_and_get_choices(storage_supplier: Callable[[], StorageSupplier]) -> None: + with storage_supplier() as storage: + study = create_study(storage=storage, n_generate=5) + for _ in range(5): + trial = study.ask() + trial.suggest_float("x", 0, 1) + + study_id = study._study._study_id + + report_history( + study_id=study_id, + storage=storage, + input_data=NewHistory( + mode="ChooseWorst", + candidates=[0, 1, 2], + clicked=1, + ), + ) + report_history( + study_id=study_id, + storage=storage, + input_data=NewHistory( + mode="ChooseWorst", + candidates=[0, 2, 3, 4], + clicked=0, + ), + ) + history = serialize_preference_history(storage.get_study_system_attrs(study_id)) + sys_attrs = storage.get_study_system_attrs(study_id) + assert len(history) == 2 + assert history[0]["candidates"] == [0, 1, 2] + assert history[0]["clicked"] == 1 + preferences = sys_attrs[_SYSTEM_ATTR_PREFIX_PREFERENCE + history[0]["preference_id"]] + assert len(preferences) == 2 + for i, (best, worst) in enumerate([(0, 1), (2, 1)]): + assert len(preferences[i]) == 2 + assert preferences[i][0] == best + assert preferences[i][1] == worst + assert history[1]["candidates"] == [0, 2, 3, 4] + assert history[1]["clicked"] == 0 + preferences = sys_attrs[_SYSTEM_ATTR_PREFIX_PREFERENCE + history[1]["preference_id"]] + assert len(preferences) == 3 + for i, (best, worst) in enumerate([(2, 0), (3, 0), (4, 0)]): + assert len(preferences[i]) == 2 + assert preferences[i][0] == best + assert preferences[i][1] == worst diff --git a/python_tests/test_serializers.py b/python_tests/test_serializers.py index a90e0de7..72db7b26 100644 --- a/python_tests/test_serializers.py +++ b/python_tests/test_serializers.py @@ -29,7 +29,7 @@ def test_get_study_detail_is_preferential() -> None: assert len(study_summaries) == 1 study_summary = study_summaries[0] - study_detail = serialize_study_detail(study_summary, [], study.trials, [], [], [], False) + study_detail = serialize_study_detail(study_summary, [], study.trials, [], [], [], False, {}) assert study_detail["is_preferential"] @@ -40,7 +40,7 @@ def test_get_study_detail_is_not_preferential() -> None: assert len(study_summaries) == 1 study_summary = study_summaries[0] - study_detail = serialize_study_detail(study_summary, [], study.trials, [], [], [], False) + study_detail = serialize_study_detail(study_summary, [], study.trials, [], [], [], False, {}) assert not study_detail["is_preferential"]