diff --git a/optuna_dashboard/preferential/_study.py b/optuna_dashboard/preferential/_study.py index 7da363b2..fb54ed36 100644 --- a/optuna_dashboard/preferential/_study.py +++ b/optuna_dashboard/preferential/_study.py @@ -14,6 +14,7 @@ from optuna.trial import FrozenTrial from optuna.trial import TrialState from optuna_dashboard.preferential._system_attrs import get_n_generate from optuna_dashboard.preferential._system_attrs import get_preferences +from optuna_dashboard.preferential._system_attrs import get_skipped_trial_ids from optuna_dashboard.preferential._system_attrs import is_skipped_trial from optuna_dashboard.preferential._system_attrs import report_preferences from optuna_dashboard.preferential._system_attrs import set_n_generate @@ -243,8 +244,11 @@ class PreferentialStudy: Returns: A list of the pair of FrozenTrial objects. The left trial is better than the right one. """ + + preferences = get_preferences( + self._study._storage.get_study_system_attrs(self._study._study_id) + ) # Must come before study.get_trials() trials = self._study.get_trials(deepcopy=deepcopy) - preferences = get_preferences(self._study._study_id, self._study._storage) return [(trials[better], trials[worse]) for (better, worse) in preferences] def set_user_attr(self, key: str, value: Any) -> None: @@ -269,11 +273,23 @@ class PreferentialStudy: to generate a new trial if this method returns :obj:`True`, and to wait for human evaluation if this method returns :obj:`False`. """ - return len(self.best_trials) < get_n_generate(self._study.system_attrs) + study_system_attrs = self._study._storage.get_study_system_attrs( + self._study._study_id + ) # Must come before _study.get_trials() + trials = self._study.get_trials(deepcopy=False) + + all_trial_ids = {t._trial_id for t in trials} + bad_trial_ids = { + trials[worse]._trial_id for (_, worse) in get_preferences(study_system_attrs) + } + skipped_trial_ids = set(get_skipped_trial_ids(study_system_attrs)) + + active_trial_ids = all_trial_ids - bad_trial_ids - skipped_trial_ids + return len(active_trial_ids) < get_n_generate(self._study.system_attrs) def get_best_trials(study_id: int, storage: optuna.storages.BaseStorage) -> list[FrozenTrial]: - preferences = get_preferences(study_id, storage) + preferences = get_preferences(storage.get_study_system_attrs(study_id)) worse_numbers = {worse for _, worse in preferences} nondominated_numbers = {better for better, _ in preferences if better not in worse_numbers} trials = storage.get_all_trials(study_id, deepcopy=False) diff --git a/optuna_dashboard/preferential/_system_attrs.py b/optuna_dashboard/preferential/_system_attrs.py index 4cb0e288..2f951594 100644 --- a/optuna_dashboard/preferential/_system_attrs.py +++ b/optuna_dashboard/preferential/_system_attrs.py @@ -35,13 +35,9 @@ def report_preferences( return preference_id -def get_preferences( - study_id: int, - storage: BaseStorage, -) -> list[tuple[int, int]]: +def get_preferences(study_system_attrs: dict[str, Any]) -> list[tuple[int, int]]: preferences: list[tuple[int, int]] = [] - system_attrs = storage.get_study_system_attrs(study_id) - for k, v in system_attrs.items(): + for k, v in study_system_attrs.items(): if not k.startswith(_SYSTEM_ATTR_PREFIX_PREFERENCE): continue preferences.extend(v) # type: ignore @@ -65,6 +61,14 @@ def is_skipped_trial(trial_id: int, study_system_attrs: dict[str, Any]) -> bool: return key in study_system_attrs +def get_skipped_trial_ids(study_system_attrs: dict[str, Any]) -> list[int]: + return [ + int(k[len(_SYSTEM_ATTR_PREFIX_SKIP_TRIAL):]) + for k in study_system_attrs.keys() + if k.startswith(_SYSTEM_ATTR_PREFIX_SKIP_TRIAL) + ] + + def get_n_generate(study_system_attrs: dict[str, Any]) -> int: return study_system_attrs[_SYSTEM_ATTR_N_GENERATE] diff --git a/optuna_dashboard/preferential/samplers/gp.py b/optuna_dashboard/preferential/samplers/gp.py index ff4001c1..b90a1de4 100644 --- a/optuna_dashboard/preferential/samplers/gp.py +++ b/optuna_dashboard/preferential/samplers/gp.py @@ -342,7 +342,7 @@ class PreferentialGPSampler(optuna.samplers.BaseSampler): if len(search_space) == 0: return {} - preferences = get_preferences(study._study_id, study._storage) + preferences = get_preferences(study.system_attrs) trials = study.get_trials(deepcopy=False) if len(preferences) == 0: return {} diff --git a/optuna_dashboard/ts/components/PreferentialTrials.tsx b/optuna_dashboard/ts/components/PreferentialTrials.tsx index 93ed2f5a..27873c45 100644 --- a/optuna_dashboard/ts/components/PreferentialTrials.tsx +++ b/optuna_dashboard/ts/components/PreferentialTrials.tsx @@ -182,11 +182,15 @@ export const PreferentialTrials: FC<{ studyDetail: StudyDetail | null }> = ({ return null } const theme = useTheme() + + const running_trials = studyDetail.trials.filter((t) => t.state === "Running") + const active_trials = running_trials.concat(studyDetail.best_trials) + const [displayTrials, setDisplayTrials] = useState({ - numbers: studyDetail.best_trials.map((t) => t.number), - last_number: Math.max(...studyDetail.best_trials.map((t) => t.number), -1), + numbers: active_trials.map((t) => t.number), + last_number: Math.max(...active_trials.map((t) => t.number), -1), }) - const new_trails = studyDetail.best_trials.filter( + const new_trails = active_trials.filter( (t) => displayTrials.last_number < t.number && displayTrials.numbers.find((n) => n === t.number) === undefined @@ -239,7 +243,7 @@ export const PreferentialTrials: FC<{ studyDetail: StudyDetail | null }> = ({ {displayTrials.numbers.map((t, index) => ( trial.number === t)} + trial={active_trials.find((trial) => trial.number === t)} candidates={displayTrials.numbers.filter((n) => n !== -1)} hideTrial={() => { hideTrial(t) diff --git a/python_tests/preferential/test_system_attrs.py b/python_tests/preferential/test_system_attrs.py index 10448d48..34f93200 100644 --- a/python_tests/preferential/test_system_attrs.py +++ b/python_tests/preferential/test_system_attrs.py @@ -18,12 +18,13 @@ def test_report_and_get_preferences(storage_supplier: Callable[[], StorageSuppli study.ask() study_id = study._study_id - assert len(get_preferences(study_id, storage)) == 0 + + assert len(get_preferences(storage.get_study_system_attrs(study_id))) == 0 better, worse = study.trials[0], study.trials[1] report_preferences(study_id, storage, [(better.number, worse.number)]) - assert len(get_preferences(study_id, storage)) == 1 + assert len(get_preferences(storage.get_study_system_attrs(study_id))) == 1 - actual_better, actual_worse = get_preferences(study_id, storage)[0] + actual_better, actual_worse = get_preferences(storage.get_study_system_attrs(study_id))[0] assert actual_better == better.number assert actual_worse == worse.number