Merge pull request #860 from porink0424/feat/tslib-react-test

Implemented tests for `@optuna/react`
2026-09-11 12:30:25 +08:00 · 2024-04-10 11:23:37 +09:00
parent e826cd7b57 de0accd7a3
commit 7e9db28d1f
12 changed files with 5173 additions and 1344 deletions
@@ -77,21 +77,33 @@ jobs:
          python-version: '3.11'
          architecture: x64

-      - name: Generate test asset
+      - name: Generate test asset for storage
        working-directory: tslib/storage/test/
        run: |
          python -m pip install --progress-bar off --upgrade pip setuptools
          pip install --progress-bar off optuna
          python generate_assets.py
+      
+      - name: Generate test asset for react
+        working-directory: tslib/react/test/
+        run: |
+          python generate_assets.py

      - name: Setup Node
        uses: actions/setup-node@v2
        with:
          node-version: '20'
          cache: 'npm'
+
      - name: Build test
        run: make tslib
-      - name: Run tslib test
+
+      - name: Run tslib test for storage
        working-directory: tslib/storage
        run: |
          npm run test
+      
+      - name: Run tslib test for react
+        working-directory: tslib/react
+        run: |
+          npm run test
@@ -44,6 +44,7 @@
      "version": "0.1.0"
    },
    "../tslib/react": {
+      "name": "@optuna/react",
      "version": "0.0.1",
      "license": "MIT",
      "dependencies": {
@@ -67,16 +68,20 @@
        "@storybook/react": "^8.0.4",
        "@storybook/react-vite": "^8.0.4",
        "@storybook/test": "^8.0.4",
+        "@testing-library/react": "^14.2.2",
        "@types/plotly.js-dist-min": "^2.3.4",
        "@types/react": "^18.2.55",
        "@types/react-dom": "^18.2.19",
        "@vitejs/plugin-react-swc": "^3.5.0",
+        "jsdom": "^24.0.0",
        "storybook": "^8.0.4",
        "typescript": "^5.2.2",
-        "vite": "^5.1.0"
+        "vite": "^5.1.0",
+        "vitest": "^1.4.0"
      }
    },
    "../tslib/storage": {
+      "name": "@optuna/storage",
      "version": "0.0.1",
      "license": "MIT",
      "dependencies": {
@@ -8,7 +8,8 @@
  "types": "types/index.d.ts",
  "scripts": {
    "build": "tsc -d",
-    "storybook": "storybook dev -p 6006"
+    "storybook": "storybook dev -p 6006",
+    "test": "vitest run"
  },
  "files": [
    "pkg",
@@ -45,12 +46,16 @@
    "@storybook/react": "^8.0.4",
    "@storybook/react-vite": "^8.0.4",
    "@storybook/test": "^8.0.4",
+    "@testing-library/jest-dom": "^6.4.2",
+    "@testing-library/react": "^14.2.2",
    "@types/plotly.js-dist-min": "^2.3.4",
    "@types/react": "^18.2.55",
    "@types/react-dom": "^18.2.19",
    "@vitejs/plugin-react-swc": "^3.5.0",
+    "jsdom": "^24.0.0",
    "storybook": "^8.0.4",
    "typescript": "^5.2.2",
-    "vite": "^5.1.0"
+    "vite": "^5.1.0",
+    "vitest": "^1.4.0"
  }
 }
@@ -0,0 +1,34 @@
+import * as Optuna from "@optuna/types"
+import { render, screen } from "@testing-library/react"
+import React from "react"
+import { describe, expect, test } from "vitest"
+import { PlotHistory } from "../src/components/PlotHistory"
+
+describe("PlotHistory Tests", async () => {
+  const setup = ({
+    study,
+    dataTestId,
+  }: { study: Optuna.Study; dataTestId: string }) => {
+    const Wrapper = ({
+      dataTestId,
+      children,
+    }: {
+      dataTestId: string
+      children: React.ReactNode
+    }) => <div data-testid={dataTestId}>{children}</div>
+    return render(
+      <Wrapper dataTestId={dataTestId}>
+        <PlotHistory study={study} />
+      </Wrapper>
+    )
+  }
+
+  for (const study of window.mockStudies) {
+    test(`PlotHistory (study name: ${study.study_name})`, () => {
+      setup({ study, dataTestId: `plot-history-${study.study_id}` })
+      expect(
+        screen.getByTestId(`plot-history-${study.study_id}`)
+      ).toBeInTheDocument()
+    })
+  }
+})
@@ -0,0 +1,34 @@
+import * as Optuna from "@optuna/types"
+import { render, screen } from "@testing-library/react"
+import React from "react"
+import { describe, expect, test } from "vitest"
+import { TrialTable } from "../src/components/TrialTable"
+
+describe("TrialTable Tests", async () => {
+  const setup = ({
+    study,
+    dataTestId,
+  }: { study: Optuna.Study; dataTestId: string }) => {
+    const Wrapper = ({
+      dataTestId,
+      children,
+    }: {
+      dataTestId: string
+      children: React.ReactNode
+    }) => <div data-testid={dataTestId}>{children}</div>
+    return render(
+      <Wrapper dataTestId={dataTestId}>
+        <TrialTable study={study} />
+      </Wrapper>
+    )
+  }
+
+  for (const study of window.mockStudies) {
+    test(`TrialTable (study name: ${study.study_name})`, () => {
+      setup({ study, dataTestId: `trial-table-${study.study_id}` })
+      expect(
+        screen.getByTestId(`trial-table-${study.study_id}`)
+      ).toBeInTheDocument()
+    })
+  }
+})
@@ -0,0 +1,325 @@
+import logging
+import math
+import os.path
+import shutil
+from typing import Tuple
+
+import optuna
+from optuna.distributions import CategoricalDistribution
+from optuna.distributions import FloatDistribution
+from optuna.storages import BaseStorage
+from optuna.storages import JournalFileStorage
+from optuna.storages import JournalStorage
+
+
+optuna.logging.set_verbosity(logging.CRITICAL)
+BASE_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "asset")
+
+
+def remove_assets() -> None:
+    if os.path.exists(BASE_DIR):
+        shutil.rmtree(BASE_DIR)
+    os.mkdir(BASE_DIR)
+
+
+def create_optuna_storage(storage: BaseStorage) -> optuna.storages.InMemoryStorage:
+    # Single-objective study
+    study = optuna.create_study(
+        study_name="single-objective", storage=storage, sampler=optuna.samplers.RandomSampler()
+    )
+
+    def objective_single(trial: optuna.Trial) -> float:
+        x1 = trial.suggest_float("x1", 0, 10)
+        x2 = trial.suggest_float("x2", 0, 10)
+        trial.suggest_categorical("x3", ["foo", "bar"])
+        return (x1 - 2) ** 2 + (x2 - 5) ** 2
+
+    study.optimize(objective_single, n_trials=100)
+
+    # Single-objective study with dynamic search space
+    study = optuna.create_study(
+        study_name="single-objective-dynamic", storage=storage, direction="maximize"
+    )
+
+    def objective_single_dynamic(trial: optuna.Trial) -> float:
+        category = trial.suggest_categorical("category", ["foo", "bar"])
+        if category == "foo":
+            return (trial.suggest_float("x1", 0, 10) - 2) ** 2
+        else:
+            return -((trial.suggest_float("x2", -10, 0) + 5) ** 2)
+
+    study.optimize(objective_single_dynamic, n_trials=50)
+
+    study = optuna.create_study(
+        study_name="check-rank-plot", storage=storage, sampler=optuna.samplers.RandomSampler()
+    )
+
+    def objective_single(trial: optuna.Trial) -> float:
+        x1 = trial.suggest_float("x1", 0, 10)
+        x2 = trial.suggest_float("x2", 0, 10)
+        trial.suggest_float("x3", 0, 10)
+        trial.suggest_float("x4", 0, 10)
+        trial.suggest_float("x5", 0, 10)
+        trial.suggest_float("x6", 0, 10)
+        return (x1 - 2) ** 2 + (x2 - 5) ** 2
+
+    study.optimize(objective_single, n_trials=1000)
+
+    # Single-objective study
+    study = optuna.create_study(study_name="single-objective-user-attrs", storage=storage)
+
+    def objective_single_user_attr(trial: optuna.Trial) -> float:
+        x1 = trial.suggest_float("x1", 0, 10)
+        x2 = trial.suggest_float("x2", 0, 10)
+        if x1 < 5:
+            trial.set_user_attr("X", "foo")
+        else:
+            trial.set_user_attr("X", "bar")
+        trial.set_user_attr("Y", x1 + x2)
+        return (x1 - 2) ** 2 + (x2 - 5) ** 2
+
+    study.optimize(objective_single_user_attr, n_trials=100)
+
+    # Single objective study with 'inf', '-inf', or 'nan' value
+    study = optuna.create_study(study_name="single-inf", storage=storage)
+
+    def objective_single_inf(trial: optuna.Trial) -> float:
+        x = trial.suggest_float("x", -10, 10)
+        if trial.number % 3 == 0:
+            return float("inf")
+        elif trial.number % 3 == 1:
+            return float("-inf")
+        else:
+            return x**2
+
+    study.optimize(objective_single_inf, n_trials=50)
+
+    # Single objective pruned after reported 'inf', '-inf', or 'nan'
+    study = optuna.create_study(study_name="single-inf-report", storage=storage)
+
+    def objective_single_inf_report(trial: optuna.Trial) -> float:
+        x = trial.suggest_float("x", -10, 10)
+        if trial.number % 3 == 0:
+            trial.report(float("inf"), 1)
+        elif trial.number % 3 == 1:
+            trial.report(float("-inf"), 1)
+        else:
+            trial.report(float("nan"), 1)
+
+        if x > 0:
+            raise optuna.TrialPruned()
+        else:
+            return x**2
+
+    study.optimize(objective_single_inf_report, n_trials=50)
+
+    # Single objective with reported nan value
+    study = optuna.create_study(study_name="single-nan-report", storage=storage)
+
+    def objective_single_nan_report(trial: optuna.Trial) -> float:
+        x1 = trial.suggest_float("x1", 0, 10)
+        x2 = trial.suggest_float("x2", 0, 10)
+        trial.report(0.5, step=0)
+        trial.report(math.nan, step=1)
+        return (x1 - 2) ** 2 + (x2 - 5) ** 2
+
+    study.optimize(objective_single_nan_report, n_trials=100)
+
+    # Single-objective study with 1 parameter
+    study = optuna.create_study(
+        study_name="single-objective-1-param", storage=storage, direction="maximize"
+    )
+
+    def objective_single_with_1param(trial: optuna.Trial) -> float:
+        x1 = trial.suggest_float("x1", 0, 10)
+        return -((x1 - 2) ** 2)
+
+    study.optimize(objective_single_with_1param, n_trials=50)
+
+    # Single-objective study with 1 parameter
+    study = optuna.create_study(study_name="long-parameter-names", storage=storage)
+
+    def objective_long_parameter_names(trial: optuna.Trial) -> float:
+        x1 = trial.suggest_float(
+            "x1_long_parameter_names_long_long_long_long_long_long_long_long_long_long", 0, 10
+        )
+        x2 = trial.suggest_float(
+            "x2_long_parameter_names_long_long_long_long_long_long_long_long_long_long", 0, 10
+        )
+        return (x1 - 2) ** 2 + (x2 - 5) ** 2
+
+    study.optimize(objective_long_parameter_names, n_trials=50)
+
+    # Multi-objective study
+    study = optuna.create_study(
+        study_name="multi-objective",
+        storage=storage,
+        directions=["minimize", "minimize"],
+    )
+    study.set_metric_names(["v0", "v1"])
+
+    def objective_multi(trial: optuna.Trial) -> Tuple[float, float]:
+        x = trial.suggest_float("x", 0, 5)
+        y = trial.suggest_float("y", 0, 3)
+        v0 = 4 * x**2 + 4 * y**2
+        v1 = (x - 5) ** 2 + (y - 5) ** 2
+        return v0, v1
+
+    study.optimize(objective_multi, n_trials=50)
+
+    # Multi-objective study with dynamic search space
+    study = optuna.create_study(
+        study_name="multi-dynamic", storage=storage, directions=["minimize", "minimize"]
+    )
+
+    def objective_multi_dynamic(trial: optuna.Trial) -> Tuple[float, float]:
+        category = trial.suggest_categorical("category", ["foo", "bar"])
+        if category == "foo":
+            x = trial.suggest_float("x1", 0, 5)
+            y = trial.suggest_float("y1", 0, 3)
+            v0 = 4 * x**2 + 4 * y**2
+            v1 = (x - 5) ** 2 + (y - 5) ** 2
+            return v0, v1
+        else:
+            x = trial.suggest_float("x2", 0, 5)
+            y = trial.suggest_float("y2", 0, 3)
+            v0 = 2 * x**2 + 2 * y**2
+            v1 = (x - 2) ** 2 + (y - 3) ** 2
+            return v0, v1
+
+    study.optimize(objective_multi_dynamic, n_trials=50)
+
+    # Pruning with no intermediate values
+    study = optuna.create_study(study_name="binh-korn-function-with-constraints", storage=storage)
+
+    def objective_prune_with_no_trials(trial: optuna.Trial) -> float:
+        x = trial.suggest_float("x", -15, 30)
+        y = trial.suggest_float("y", -15, 30)
+        v = x**2 + y**2
+        if v > 100:
+            raise optuna.TrialPruned()
+        return v
+
+    study.optimize(objective_prune_with_no_trials, n_trials=100)
+
+    # With failed trials
+    study = optuna.create_study(study_name="failed trials", storage=storage)
+
+    def objective_sometimes_got_failed(trial: optuna.Trial) -> float:
+        x = trial.suggest_float("x", -15, 30)
+        y = trial.suggest_float("y", -15, 30)
+        v = x**2 + y**2
+        if v > 100:
+            raise ValueError("unexpected error")
+        return v
+
+    study.optimize(objective_sometimes_got_failed, n_trials=100, catch=(Exception,))
+
+    # No trials single-objective study
+    study = optuna.create_study(study_name="no trials single-objective study", storage=storage)
+    study.set_user_attr("foo", "bar")
+
+    # study with waiting trials
+    study = optuna.create_study(study_name="waiting-trials", storage=storage)
+    study.enqueue_trial({"x": 0, "y": 10})
+    study.enqueue_trial({"x": 10, "y": 20})
+
+    # Study with Running Trials
+    study = optuna.create_study(
+        study_name="running-trials", storage=storage, directions=["minimize", "maximize"]
+    )
+    study.set_metric_names(["auc", "val_loss"])
+    study.enqueue_trial({"x": 10, "y": "Foo"})
+    study.ask({"x": FloatDistribution(0, 10), "y": CategoricalDistribution(["Foo", "Bar"])})
+    study.ask({"x": FloatDistribution(0, 10), "y": CategoricalDistribution(["Foo", "Bar"])})
+
+    # Single-objective study with constraints
+    def constraints(trial: optuna.Trial) -> list[float]:
+        return trial.user_attrs["constraint"]
+
+    study = optuna.create_study(
+        study_name="A single objective constraint optimization study",
+        storage=storage,
+        sampler=optuna.samplers.TPESampler(constraints_func=constraints),
+    )
+
+    def objective_constraints(trial: optuna.Trial) -> float:
+        x = trial.suggest_float("x", -15, 30)
+        y = trial.suggest_float("y", -15, 30)
+        v0 = 4 * x**2 + 4 * y**2
+        trial.set_user_attr("constraint", [1000 - v0, x - 10, y - 10])
+        return v0
+
+    study.optimize(objective_constraints, n_trials=100)
+
+    # Study with Running Trials
+    study = optuna.create_study(
+        study_name="objective-form-widgets",
+        storage=storage,
+        directions=["minimize", "minimize", "minimize", "minimize"],
+    )
+    study.set_metric_names(
+        ["Slider Objective", "Good or Bad", "Text Input Objective", "Validation Loss"]
+    )
+    trial = study.ask(
+        {"x": FloatDistribution(0, 10), "y": CategoricalDistribution(["Foo", "Bar"])}
+    )
+    trial.set_user_attr("val_loss", 0.2)
+    study.ask({"x": FloatDistribution(0, 10), "y": CategoricalDistribution(["Foo", "Bar"])})
+    trial.set_user_attr("val_loss", 0.5)
+
+    # No trials multi-objective study
+    optuna.create_study(
+        study_name="no trials multi-objective study",
+        storage=storage,
+        directions=["minimize", "maximize"],
+    )
+
+    # Single-objective study with intermediate values
+    study = optuna.create_study(study_name="intermediate-values", storage=storage)
+
+    def objective_intermediate_values(trial: optuna.Trial) -> float:
+        trial.report(trial.number, step=0)
+        trial.report(trial.number + 1, step=1)
+        return 0.0
+
+    study.optimize(objective_intermediate_values, n_trials=10)
+    trial = study.ask(
+        {"x": FloatDistribution(0, 10), "y": CategoricalDistribution(["Foo", "Bar"])}
+    )  # To create a running trial
+    trial.report(trial.number, step=0)
+    trial.report(trial.number + 1, step=1)
+
+    # Single-objective study with intermediate values and constraints
+    def constraints(trial: optuna.Trial) -> list[float]:
+        return trial.user_attrs["constraint"]
+
+    study = optuna.create_study(
+        study_name="intermediate-values-constraints",
+        storage=storage,
+        sampler=optuna.samplers.NSGAIISampler(constraints_func=constraints),
+    )
+
+    def objective_intermediate_values_constraints(trial: optuna.Trial) -> float:
+        trial.set_user_attr("constraint", [trial.number % 2])
+
+        trial.report(trial.number, step=0)
+        trial.report(trial.number + 1, step=1)
+        return 0.0
+
+    study.optimize(objective_intermediate_values_constraints, n_trials=10)
+    trial = study.ask(
+        {"x": FloatDistribution(0, 10), "y": CategoricalDistribution(["Foo", "Bar"])}
+    )  # To create a running trial
+    trial.report(trial.number, step=0)
+    trial.report(trial.number + 1, step=1)
+
+
+def main() -> None:
+    remove_assets()
+    storage = JournalStorage(JournalFileStorage(os.path.join(BASE_DIR, "journal.log")))
+    create_optuna_storage(storage)
+
+
+if __name__ == "__main__":
+    main()
@@ -0,0 +1,26 @@
+import fs from "node:fs"
+import * as Optuna from "@optuna/types"
+import { loadStorageFromFile } from "../src/utils/loadStorageFromFile"
+
+declare global {
+  interface Window {
+    mockStudies: Optuna.Study[]
+  }
+}
+
+const data = fs.readFileSync("./test/asset/journal.log")
+const blob = new Blob([data])
+const file = new File([blob], "journal.log")
+const mockStudies: Optuna.Study[] = []
+await loadStorageFromFile(file, (value) => {
+  if (Array.isArray(value)) {
+    mockStudies.push(...value)
+  } else {
+    mockStudies.push(...value([]))
+  }
+})
+window.mockStudies = mockStudies
+
+// mock window.URL.createObjectURL in JSDOM
+window.HTMLCanvasElement.prototype.getContext = () => null
+window.URL.createObjectURL = () => ""
@@ -0,0 +1,22 @@
+{
+  "compilerOptions": {
+    "isolatedModules": true,
+    "skipLibCheck": true,
+    "strictNullChecks": true,
+    "moduleResolution": "Node",
+    "noUnusedLocals": true,
+    "noImplicitThis": true,
+    "alwaysStrict": true,
+    "noImplicitAny": true,
+    "lib": ["ES2020", "DOM", "DOM.Iterable"],
+    "module": "ESNext",
+    "esModuleInterop": true,
+    "allowSyntheticDefaultImports": true,
+    "target": "ES2020",
+    "strict": true,
+    "noUnusedParameters": true,
+    "noFallthroughCasesInSwitch": true,
+    "jsx": "react-jsx"
+  },
+  "include": ["**/*.ts", "**/*.tsx"]
+}
@@ -0,0 +1,9 @@
+import "@testing-library/jest-dom/vitest"
+
+declare global {
+  namespace jest {
+    interface Matchers<R> {
+      toBeInTheDocument(): R
+    }
+  }
+}
@@ -1,9 +1,18 @@
 import react from "@vitejs/plugin-react-swc"
-import { defineConfig } from "vite"
+import { UserConfig, defineConfig } from "vite"
+import { InlineConfig } from "vitest"
+
+interface VitestConfig extends UserConfig {
+  test: InlineConfig
+}

 // https://vitejs.dev/config/
 export default defineConfig({
  plugins: [react()],
+  test: {
+    environment: "jsdom",
+    setupFiles: ["./test/vitest_setup.ts", "./test/setup_studies.ts"],
+  },
  optimizeDeps: {
    exclude: ["@sqlite.org/sqlite-wasm"],
  },
@@ -13,4 +22,4 @@ export default defineConfig({
      "Cross-Origin-Embedder-Policy": "require-corp",
    },
  },
-})
+}) as VitestConfig
@@ -289,7 +289,7 @@ class JournalStorage {
      return
    }
    thisTrial.state = trialStateNumToTrialState(log.state)
-    thisTrial.values = log.values
+    thisTrial.values = log.values === null ? undefined : log.values
    thisTrial.datetime_start = log.datetime_start
      ? new Date(log.datetime_start)
      : undefined
@@ -340,7 +340,31 @@ const loadJournalStorage = (arrayBuffer: ArrayBuffer): Optuna.Study[] => {
    if (log === "") {
      continue
    }
-    const parsedLog: JournalOpBase = JSON.parse(log)
+
+    const parsedLog: JournalOpBase = (() => {
+      try {
+        return JSON.parse(log)
+      } catch (error) {
+        if (error instanceof SyntaxError) {
+          let escapedLog: string = log.replace(/NaN/g, '"***nan***"')
+          escapedLog = escapedLog.replace(/-Infinity/g, '"***-inf***"')
+          escapedLog = escapedLog.replace(/Infinity/g, '"***inf***"')
+          return JSON.parse(escapedLog, (_key, value) => {
+            switch (value) {
+              case "***nan***":
+                return NaN
+              case "***-inf***":
+                return -Infinity
+              case "***inf***":
+                return Infinity
+              default:
+                return value
+            }
+          })
+        }
+      }
+    })()
+
    switch (parsedLog.op_code) {
      case JournalOperation.CREATE_STUDY:
        journalStorage.applyCreateStudy(parsedLog as JournalOpCreateStudy)