Merge branch 'main' into optional-dependencies

.github/workflows/stale.yml

@@ -24,6 +24,6 @@ jobs:
         days-before-pr-close: 7  # default number
         stale-issue-label: 'stale'
         stale-pr-label: 'stale'
-        exempt-issue-labels: 'no-stale'
+        exempt-issue-labels: 'no-stale,good-first-issue,contribution-welcome'
         exempt-pr-labels: 'no-stale'
         operations-per-run: 1000

optuna_dashboard/preferential/_study.py

@@ -37,6 +37,10 @@ class PreferentialStudy:
     To create and load a study, please refer to the documentation of
     :func:`~optuna_dashboard.preferential.create_study` and
     :func:`~optuna_dashboard.preferential.load_study` respectively.
+
+    .. note::
+        Preferential optimization is an experimental feature (introduced in v0.13.0).
+        The interface may change in newer versions without prior notice.
     """
 
     def __init__(self, study: optuna.Study) -> None:
@@ -359,6 +363,10 @@ def create_study(
 
     Returns:
         A :class:`~optuna_dashboard.preferential.PreferentialStudy` object.
+
+    .. note::
+        Preferential optimization is an experimental feature (introduced in v0.13.0).
+        The interface may change in newer versions without prior notice.
     """
     try:
         study = optuna.create_study(
@@ -441,6 +449,10 @@ def load_study(
 
     Returns:
         A :class:`~optuna_dashboard.preferential.PreferentialStudy` object.
+
+    .. note::
+        Preferential optimization is an experimental feature (introduced in v0.13.0).
+        The interface may change in newer versions without prior notice.
     """
     study = optuna.load_study(
         study_name=study_name, storage=storage, sampler=sampler or RandomSampler()

optuna_dashboard/preferential/samplers/gp.py

@@ -4,7 +4,7 @@ import itertools
 import math
 from typing import Any
 from typing import Callable
-from typing import cast
+import warnings
 
 import botorch.acquisition.analytic
 import botorch.models.model
@@ -17,6 +17,8 @@ import numpy as np
 import optuna
 import optuna._transform
 from optuna.distributions import CategoricalDistribution
+from optuna.distributions import FloatDistribution
+from optuna.distributions import IntDistribution
 import torch
 from torch import Tensor
 
@@ -156,6 +158,18 @@ def _truncnorm_mean_var_logz(alpha: Tensor) -> tuple[Tensor, Tensor, Tensor]:
     return mean, var, logz
 
 
+def _observation(var0: Tensor, mean0: Tensor, noise_var: Tensor) -> tuple[Tensor, Tensor, Tensor]:
+    obs_var = var0 + noise_var
+    obs_sigma = torch.sqrt(obs_var)
+    alpha = -mean0 / torch.clamp_min(obs_sigma, min=1e-20)
+    mean_norm, var_norm, logz = _truncnorm_mean_var_logz(alpha)
+
+    denom_factor = 1 / torch.clamp_min(noise_var + var_norm * var0, min=1e-20)
+    da = (1 - var_norm) * denom_factor
+    db = (mean0 * (1 - var_norm) + obs_sigma * mean_norm) * denom_factor
+    return (da, db, logz)
+
+
 def _orthants_MVN_EP(
     cov0: Tensor, preferences: Tensor, noise_var: Tensor, cycles: int
 ) -> tuple[Tensor, Tensor, Tensor]:
@@ -176,25 +190,17 @@ def _orthants_MVN_EP(
 
             r0 = (1 - var1 * virtual_obs_a[i]).reciprocal()
             var0 = var1 * r0
-            mean0 = (mean1 + var1 * virtual_obs_b[i]) * r0
+            mean0 = (mean1 - var1 * virtual_obs_b[i]) * r0
 
-            obs_var = var0 + noise_var
-            obs_sigma = torch.sqrt(obs_var)
-            alpha = -mean0 / torch.clamp_min(obs_sigma, min=1e-20)
-            mean_norm, var_norm, logz = _truncnorm_mean_var_logz(alpha)
+            virtual_obs_a2, virtual_obs_b2, logz = _observation(var0, mean0, noise_var)
 
-            kalman_factor = var0 / torch.clamp_min(obs_var, min=1e-20)
-            mean2 = mean0 + obs_sigma * mean_norm * kalman_factor
-            var2 = kalman_factor * (noise_var + var_norm * var0)
-
-            var1_var2_inv = torch.clamp_min(var1 * var2, min=1e-20).reciprocal()
-            db = (mean1 * var2 - mean2 * var1) * var1_var2_inv
-            da = (var1 - var2) * var1_var2_inv
-            virtual_obs_b[i] = virtual_obs_b[i] + db
-            virtual_obs_a[i] = virtual_obs_a[i] + da
+            da = virtual_obs_a2 - virtual_obs_a[i]
+            db = virtual_obs_b2 - virtual_obs_b[i]
+            virtual_obs_a[i] = virtual_obs_a2
+            virtual_obs_b[i] = virtual_obs_b2
 
             dr = (1 + var1 * da).reciprocal()
-            mu = mu - Sxy * ((db + mean1 * da) * dr)
+            mu = mu + Sxy * ((db - mean1 * da) * dr)
             cov = cov - (Sxy[:, None] * (da * dr)) @ Sxy[None, :]
             log_zs[i] = logz
     return mu, cov, torch.sum(log_zs)
@@ -359,11 +365,54 @@ class PreferentialGPSampler(optuna.samplers.BaseSampler):
             )
 
             # TODO: Make it possible to apply it on mixed search space
-            if all(isinstance(dist, CategoricalDistribution) for dist in search_space.values()):
+            def get_all_possible_params(dist: optuna.distributions.BaseDistribution) -> list[Any]:
+                if isinstance(dist, CategoricalDistribution):
+                    return list(dist.choices)
+                elif isinstance(dist, (IntDistribution, FloatDistribution)):
+                    return list(np.arange(dist.low, dist.high, dist.step))
+                else:
+                    return []
+
+            all_possible_params = {
+                name: get_all_possible_params(dist) for name, dist in search_space.items()
+            }
+
+            is_all_discrete = all(
+                len(possible_params) > 0 for possible_params in all_possible_params.values()
+            )
+            search_space_size = np.prod(
+                [len(possible_params) for possible_params in all_possible_params.values()]
+            )
+            # TODO(contramundum53): Fix this arbitrarily chosen limit.
+            size_limit = 1e6
+            can_evaluate_all = is_all_discrete and search_space_size <= size_limit
+
+            if (
+                any(isinstance(dist, CategoricalDistribution) for dist in search_space.values())
+                and not can_evaluate_all
+            ):
+                if is_all_discrete:
+                    warnings.warn(
+                        "The objective function has categorical parameters, "
+                        "but the total search space is too large to be enumerated. "
+                        f"(Search space size: {search_space_size} > limit: {size_limit})"
+                        "This may result in significantly bad performance."
+                    )
+                else:
+                    warnings.warn(
+                        "The objective function has categorical parameters, "
+                        "but the search space cannot be enumerated because "
+                        "it also contains continuous parameters. "
+                        "This may result in significantly bad performance. "
+                        "You can work around this problem by specifying 'step' "
+                        "in each continuous parameter."
+                    )
+
+            if is_all_discrete and can_evaluate_all:
                 all_param_combinations = itertools.product(
                     *[
-                        [(name, choice) for choice in cast(CategoricalDistribution, dist).choices]
-                        for name, dist in search_space.items()
+                        [(name, choice) for choice in possible_params]
+                        for name, possible_params in all_possible_params.items()
                     ]
                 )
                 choices = torch.tensor(
@@ -395,6 +444,11 @@ class PreferentialGPSampler(optuna.samplers.BaseSampler):
         param_name: str,
         param_distribution: optuna.distributions.BaseDistribution,
     ) -> Any:
+        warnings.warn(
+            "Dynamic search space detected. "
+            f"Falling back to {self.independent_sampler.__class__.__name__}."
+        )
+
         return self.independent_sampler.sample_independent(
             study, trial, param_name, param_distribution
         )

optuna_dashboard/ts/components/AppDrawer.tsx

@@ -17,7 +17,7 @@ import ListItemText from "@mui/material/ListItemText"
 import {
   drawerOpenState,
   reloadIntervalState,
-  useStudyIsPreferencial,
+  useStudyIsPreferential,
 } from "../state"
 import { Link } from "react-router-dom"
 import AutoGraphIcon from "@mui/icons-material/AutoGraph"
@@ -130,7 +130,7 @@ export const AppDrawer: FC<{
   const [open, setOpen] = useRecoilState<boolean>(drawerOpenState)
   const reloadInterval = useRecoilValue<number>(reloadIntervalState)
   const isPreferential =
-    studyId !== undefined ? useStudyIsPreferencial(studyId) : null
+    studyId !== undefined ? useStudyIsPreferential(studyId) : null
 
   const styleListItem = {
     display: "block",

optuna_dashboard/ts/components/ArtifactCardMedia.tsx

@@ -21,6 +21,18 @@ export const ArtifactCardMedia: FC<{
         filetype={artifact.filename.split(".").pop()}
       />
     )
+  } else if (artifact.mimetype.startsWith("video")) {
+    return (
+      <video
+        controls
+        style={{
+          width: "100%",
+          height: "auto",
+        }}
+      >
+        <source src={urlPath} type={artifact.mimetype} />
+      </video>
+    )
   } else if (artifact.mimetype.startsWith("audio")) {
     return (
       <audio controls>

optuna_dashboard/ts/components/GraphContour.tsx

@@ -162,19 +162,19 @@ const plotContour = (
     return
   }
 
-  const xAxis = getAxisInfo(study, trials, xParam)
-  const yAxis = getAxisInfo(study, trials, yParam)
+  const xAxis = getAxisInfo(trials, xParam)
+  const yAxis = getAxisInfo(trials, yParam)
   const xIndices = xAxis.indices
   const yIndices = yAxis.indices
 
   const layout: Partial<plotly.Layout> = {
     xaxis: {
       title: xParam.name,
-      type: xAxis.isCat ? "category" : undefined,
+      type: xAxis.isCat ? "category" : xAxis.isLog ? "log" : "linear",
     },
     yaxis: {
       title: yParam.name,
-      type: yAxis.isCat ? "category" : undefined,
+      type: yAxis.isCat ? "category" : yAxis.isLog ? "log" : "linear",
     },
     margin: {
       l: 50,
@@ -278,9 +278,19 @@ const getAxisInfoForNumericalParams = (
   paramName: string,
   distribution: FloatDistribution | IntDistribution
 ): AxisInfo => {
-  const padding = (distribution.high - distribution.low) * PADDING_RATIO
-  const min = distribution.low - padding
-  const max = distribution.high + padding
+  let min = 0
+  let max = 0
+  if (distribution.log) {
+    const padding =
+      (Math.log10(distribution.high) - Math.log10(distribution.low)) *
+      PADDING_RATIO
+    min = Math.pow(10, Math.log10(distribution.low) - padding)
+    max = Math.pow(10, Math.log10(distribution.high) + padding)
+  } else {
+    const padding = (distribution.high - distribution.low) * PADDING_RATIO
+    min = distribution.low - padding
+    max = distribution.high + padding
+  }
 
   const values = trials.map(
     (trial) =>
@@ -341,11 +351,7 @@ const getAxisInfoForCategoricalParams = (
   }
 }
 
-const getAxisInfo = (
-  study: StudyDetail,
-  trials: Trial[],
-  param: SearchSpaceItem
-): AxisInfo => {
+const getAxisInfo = (trials: Trial[], param: SearchSpaceItem): AxisInfo => {
   if (param.distribution.type === "CategoricalDistribution") {
     return getAxisInfoForCategoricalParams(
       trials,

optuna_dashboard/ts/components/GraphParallelCoordinate.tsx

@@ -169,6 +169,21 @@ const plotCoordinate = (
       .join("")
   }
 
+  const calculateLogScale = (values: number[]) => {
+    const logValues = values.map((v) => {
+      return Math.log10(v)
+    })
+    const minValue = Math.min(...logValues)
+    const maxValue = Math.max(...logValues)
+    const range = [Math.floor(minValue), Math.ceil(maxValue)]
+    const tickvals = Array.from(
+      { length: Math.ceil(maxValue) - Math.floor(minValue) + 1 },
+      (_, i) => i + Math.floor(minValue)
+    )
+    const ticktext = tickvals.map((x) => `${Math.pow(10, x).toPrecision(3)}`)
+    return { logValues, range, tickvals, ticktext }
+  }
+
   const dimensions = targets.map((target) => {
     if (target.kind === "objective" || target.kind === "user_attr") {
       const values: number[] = trials.map(
@@ -187,13 +202,7 @@ const plotCoordinate = (
       const values: number[] = trials.map(
         (t) => target.getTargetValue(t) as number
       )
-      if (s.distribution.type !== "CategoricalDistribution") {
-        return {
-          label: breakLabelIfTooLong(s.name),
-          values: values,
-          range: [s.distribution.low, s.distribution.high],
-        }
-      } else {
+      if (s.distribution.type === "CategoricalDistribution") {
         // categorical
         const vocabArr: string[] = s.distribution.choices.map((c) => c.value)
         const tickvals: number[] = vocabArr.map((v, i) => i)
@@ -205,6 +214,24 @@ const plotCoordinate = (
           tickvals: tickvals,
           ticktext: vocabArr,
         }
+      } else if (s.distribution.log) {
+        // numerical and log
+        const { logValues, range, tickvals, ticktext } =
+          calculateLogScale(values)
+        return {
+          label: breakLabelIfTooLong(s.name),
+          values: logValues,
+          range,
+          tickvals,
+          ticktext,
+        }
+      } else {
+        // numerical and linear
+        return {
+          label: breakLabelIfTooLong(s.name),
+          values: values,
+          range: [s.distribution.low, s.distribution.high],
+        }
       }
     }
   })

optuna_dashboard/ts/components/StudyDetail.tsx

@@ -18,7 +18,7 @@ import { actionCreator } from "../action"
 import {
   reloadIntervalState,
   useStudyDetailValue,
-  useStudyIsPreferencial,
+  useStudyIsPreferential,
   useStudyName,
 } from "../state"
 import { TrialTable } from "./TrialTable"
@@ -54,7 +54,7 @@ export const StudyDetail: FC<{
   const studyDetail = useStudyDetailValue(studyId)
   const reloadInterval = useRecoilValue<number>(reloadIntervalState)
   const studyName = useStudyName(studyId)
-  const isPreferential = useStudyIsPreferencial(studyId)
+  const isPreferential = useStudyIsPreferential(studyId)
 
   const title =
     studyName !== null ? `${studyName} (id=${studyId})` : `Study #${studyId}`

optuna_dashboard/ts/state.ts

@@ -87,7 +87,7 @@ export const useStudyDirections = (
   return studyDetail?.directions || studySummary?.directions || null
 }
 
-export const useStudyIsPreferencial = (studyId: number): boolean | null => {
+export const useStudyIsPreferential = (studyId: number): boolean | null => {
   const studyDetail = useStudyDetailValue(studyId)
   const studySummary = useStudySummaryValue(studyId)
   return studyDetail?.is_preferential || studySummary?.is_preferential || null

standalone_app/src/sqlite3.ts

@@ -38,10 +38,11 @@ export const loadStorage = (
     )
     db.checkRc(rc)
     try {
-      if (!isSupportedSchema(db)) {
+      const schemaVersion = getSchemaVersion(db)
+      if (!isSupportedSchema(schemaVersion)) {
         return
       }
-      const studies = getStudies(db)
+      const studies = getStudies(db, schemaVersion)
       setter((prev) => [...prev, ...studies])
     } finally {
       db.close()
@@ -49,21 +50,41 @@ export const loadStorage = (
   })
 }
 
-const isSupportedSchema = (db: SQLite3DB): boolean => {
-  let supported = true
+const getSchemaVersion = (db: SQLite3DB): string => {
+  let schemaVersion = ""
   db.exec({
-    sql: "SELECT schema_version FROM version_info LIMIT 1",
+    sql: "SELECT version_num FROM alembic_version LIMIT 1",
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
     callback: (vals: any[]) => {
-      if (vals[0] != 12) {
-        supported = false
-      }
+      schemaVersion = vals[0]
     },
   })
-  return supported
+  return schemaVersion
 }
 
-const getStudies = (db: SQLite3DB): Study[] => {
+const isSupportedSchema = (schemaVersion: string): boolean => {
+  const lowestVersion = "v2.6.0.a" // supported: "v3.2.0.a", "v3.0.0.{a,b,c,d}", "v2.6.0.a"
+  if (schemaVersion == lowestVersion) return true
+  return isGreaterSchemaVersion(schemaVersion, lowestVersion)
+}
+
+const isGreaterSchemaVersion = (
+  leftVersion: string,
+  rightVersion: string
+): boolean => {
+  // return leftVersion > rightVersion
+  const leftSuffix = leftVersion.split(".").reverse()[0]
+  const rightSuffix = rightVersion.split(".").reverse()[0]
+  leftVersion = leftVersion.replace(/\D/g, "")
+  rightVersion = rightVersion.replace(/\D/g, "")
+
+  const left = Number(leftVersion)
+  const right = Number(rightVersion)
+  if (left == right) return leftSuffix > rightSuffix
+  return left > right
+}
+
+const getStudies = (db: SQLite3DB, schemaVersion: string): Study[] => {
   const studies: Study[] = []
   db.exec({
     sql:
@@ -78,7 +99,7 @@ const getStudies = (db: SQLite3DB): Study[] => {
         vals[2] === "MINIMIZE" ? "minimize" : "maximize"
       const objective = vals[3]
 
-      const trials = getTrials(db, studyId)
+      const trials = getTrials(db, studyId, schemaVersion)
       const union_search_space: SearchSpaceItem[] = []
       const union_user_attrs: AttributeSpec[] = []
       let intersection_search_space: Set<SearchSpaceItem> = new Set()
@@ -136,7 +157,11 @@ const getStudies = (db: SQLite3DB): Study[] => {
   return studies
 }
 
-const getTrials = (db: SQLite3DB, studyId: number): Trial[] => {
+const getTrials = (
+  db: SQLite3DB,
+  studyId: number,
+  schemaVersion: string
+): Trial[] => {
   const trials: Trial[] = []
   db.exec({
     sql:
@@ -160,8 +185,12 @@ const getTrials = (db: SQLite3DB, studyId: number): Trial[] => {
         number: vals[1],
         study_id: studyId,
         state: state,
-        values: getTrialValues(db, trialId),
-        intermediate_values: getTrialIntermediateValues(db, trialId),
+        values: getTrialValues(db, trialId, schemaVersion),
+        intermediate_values: getTrialIntermediateValues(
+          db,
+          trialId,
+          schemaVersion
+        ),
         params: [], // Set this column later
         user_attrs: [], // Set this column later
         datetime_start: vals[3],
@@ -173,24 +202,41 @@ const getTrials = (db: SQLite3DB, studyId: number): Trial[] => {
   return trials
 }
 
-const getTrialValues = (db: SQLite3DB, trialId: number): TrialValueNumber[] => {
+const getTrialValues = (
+  db: SQLite3DB,
+  trialId: number,
+  schemaVersion: string
+): TrialValueNumber[] => {
   const values: TrialValueNumber[] = []
-  db.exec({
-    sql:
-      "SELECT value, value_type" +
-      ` FROM trial_values WHERE trial_id = ${trialId}` +
-      " ORDER BY objective",
-    // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    callback: (vals: any[]) => {
-      values.push(
-        vals[1] === "INF_NEG"
-          ? "-inf"
-          : vals[1] === "INF_POS"
-          ? "+inf"
-          : vals[0]
-      )
-    },
-  })
+  if (isGreaterSchemaVersion(schemaVersion, "v3.0.0.c")) {
+    db.exec({
+      sql:
+        "SELECT value, value_type" +
+        ` FROM trial_values WHERE trial_id = ${trialId}` +
+        " ORDER BY objective",
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      callback: (vals: any[]) => {
+        values.push(
+          vals[1] === "INF_NEG"
+            ? "-inf"
+            : vals[1] === "INF_POS"
+            ? "+inf"
+            : vals[0]
+        )
+      },
+    })
+  } else {
+    db.exec({
+      sql:
+        "SELECT value" +
+        ` FROM trial_values WHERE trial_id = ${trialId}` +
+        " ORDER BY objective",
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      callback: (vals: any[]) => {
+        values.push(vals[0])
+      },
+    })
+  }
   return values
 }
 
@@ -241,6 +287,30 @@ const parseDistributionJSON = (t: string): Distribution => {
       step: parsed.attributes.step as number,
       log: parsed.attributes.log as boolean,
     }
+  } else if (parsed.name === "UniformDistribution") {
+    return {
+      type: "FloatDistribution",
+      low: parsed.attributes.low as number,
+      high: parsed.attributes.high as number,
+      step: null,
+      log: false,
+    }
+  } else if (parsed.name === "LogUniformDistribution") {
+    return {
+      type: "FloatDistribution",
+      low: parsed.attributes.low as number,
+      high: parsed.attributes.high as number,
+      step: null,
+      log: true,
+    }
+  } else if (parsed.name === "DiscreteUniformDistribution") {
+    return {
+      type: "FloatDistribution",
+      low: parsed.attributes.low as number,
+      high: parsed.attributes.high as number,
+      step: parsed.attributes.q,
+      log: false,
+    }
   } else if (parsed.name === "IntDistribution") {
     return {
       type: "IntDistribution",
@@ -249,6 +319,22 @@ const parseDistributionJSON = (t: string): Distribution => {
       step: parsed.attributes.step as number,
       log: parsed.attributes.log as boolean,
     }
+  } else if (parsed.name === "IntUniformDistribution") {
+    return {
+      type: "IntDistribution",
+      low: parsed.attributes.low as number,
+      high: parsed.attributes.high as number,
+      step: parsed.attributes.step as number,
+      log: false,
+    }
+  } else if (parsed.name === "IntLogUniformDistribution") {
+    return {
+      type: "IntDistribution",
+      low: parsed.attributes.low as number,
+      high: parsed.attributes.high as number,
+      step: parsed.attributes.step as number,
+      log: true,
+    }
   } else {
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
     const choices = parsed.attributes.choices.map((value: any) => {
@@ -287,26 +373,45 @@ const getTrialUserAttributes = (
 
 const getTrialIntermediateValues = (
   db: SQLite3DB,
-  trialId: number
+  trialId: number,
+  schemaVersion: string
 ): TrialIntermediateValue[] => {
   const values: TrialIntermediateValue[] = []
-  db.exec({
-    sql:
-      "SELECT step, intermediate_value, intermediate_value_type" +
-      ` FROM trial_intermediate_values WHERE trial_id = ${trialId}` +
-      " ORDER BY step",
-    // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    callback: (vals: any[]) => {
-      values.push({
-        step: vals[0],
-        value:
-          vals[2] === "INF_NEG"
-            ? "-inf"
-            : vals[2] === "INF_POS"
-            ? "+inf"
-            : vals[1],
-      })
-    },
-  })
+  if (isGreaterSchemaVersion(schemaVersion, "v3.0.0.c")) {
+    db.exec({
+      sql:
+        "SELECT step, intermediate_value, intermediate_value_type" +
+        ` FROM trial_intermediate_values WHERE trial_id = ${trialId}` +
+        " ORDER BY step",
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      callback: (vals: any[]) => {
+        values.push({
+          step: vals[0],
+          value:
+            vals[2] === "INF_NEG"
+              ? "-inf"
+              : vals[2] === "INF_POS"
+              ? "+inf"
+              : vals[2] === "NAN"
+              ? "nan"
+              : vals[1],
+        })
+      },
+    })
+  } else {
+    db.exec({
+      sql:
+        "SELECT step, intermediate_value" +
+        ` FROM trial_intermediate_values WHERE trial_id = ${trialId}` +
+        " ORDER BY step",
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      callback: (vals: any[]) => {
+        values.push({
+          step: vals[0],
+          value: vals[1],
+        })
+      },
+    })
+  }
   return values
 }

standalone_app/src/types/index.d.ts

@@ -10,7 +10,7 @@ type FloatDistribution = {
   type: "FloatDistribution"
   low: number
   high: number
-  step: number
+  step: number | null
   log: boolean
 }
 
@@ -18,7 +18,7 @@ type IntDistribution = {
   type: "IntDistribution"
   low: number
   high: number
-  step: number
+  step: number | null
   log: boolean
 }