Merge pull request #653 from keisuke-umezawa/feature/add-standalone-db-test

Add standalone e2e tests

.github/workflows/e2e-dashboard-tests.yml

@@ -1,10 +1,10 @@
-name: e2e-tests
+name: e2e-dashboard-tests
 on:
   pull_request:
     branches:
       - main
     paths:
-      - '.github/workflows/e2e-tests.yml'
+      - '.github/workflows/e2e-dashboard-tests.yml'
       - '**.py'
       - '**.ts'
       - '**.tsx'
@@ -47,4 +47,4 @@ jobs:
         run: playwright install
 
       - name: Run e2e tests
-        run: pytest e2e_tests
+        run: pytest e2e_tests/test_dashboard

.github/workflows/e2e-standalone-tests.yml

@@ -0,0 +1,51 @@
+name: e2e-standalone-tests
+on:
+  pull_request:
+    branches:
+      - main
+    paths:
+      - '.github/workflows/e2e-standalone-tests.yml'
+      - '**.py'
+      - '**.ts'
+      - '**.tsx'
+      - 'package.json'
+      - 'package-lock.json'
+      - 'tsconfig.json'
+jobs:
+  test:
+    runs-on: ubuntu-20.04
+    strategy:
+      matrix:
+        optuna-version: ['optuna==2.10.0', 'git+https://github.com/optuna/optuna.git']
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Install Rust toolchains
+        uses: dtolnay/rust-toolchain@stable
+
+      - name: Install wasm-pack
+        run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh
+
+      - name: Setup Node
+        uses: actions/setup-node@v2
+        with:
+          node-version: '18'
+
+      - name: Setup Optuna ${{ matrix.optuna-version }}
+        run: |
+          python -m pip install --progress-bar off --upgrade pip setuptools
+          python -m pip install --progress-bar off --upgrade ${{ matrix.optuna-version }}
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --progress-bar off .
+          python -m pip install --progress-bar off pytest-playwright
+
+      - name: Build standalone_app
+        run: make MODE="prd" standalone_app/public/bundle.js
+
+      - name: Install the required browsers
+        run: playwright install
+
+      - name: Run e2e tests
+        run: pytest e2e_tests/test_standalone

e2e_tests/test_dashboard/test_usecases/test_study_history.py

@@ -2,7 +2,7 @@ import optuna
 from playwright.sync_api import Page
 import pytest
 
-from ..test_server import make_test_server
+from ...test_server import make_test_server
 
 
 def make_test_storage() -> optuna.storages.InMemoryStorage:

e2e_tests/test_dashboard/visual_regression_test.py

@@ -4,7 +4,7 @@ import optuna
 from playwright.sync_api import Page
 import pytest
 
-from .test_server import make_test_server
+from ..test_server import make_test_server
 
 
 @pytest.fixture

e2e_tests/test_server.py

@@ -1,4 +1,6 @@
+import http.server
 import socket
+import socketserver
 import threading
 from wsgiref.simple_server import make_server
 
@@ -33,3 +35,29 @@ def make_test_server(
     request.addfinalizer(stop_server)
 
     return f"http://{addr}:{port}/dashboard"
+
+
+def make_standalone_server(request: pytest.FixtureRequest) -> str:
+    addr = "127.0.0.1"
+    port = get_free_port()
+    directory = "./standalone_app/"
+
+    Handler = http.server.SimpleHTTPRequestHandler
+    httpd = socketserver.TCPServer(
+        ("", port), lambda *args, **kwargs: Handler(*args, directory=directory, **kwargs)
+    )
+
+    def serve_httpd():
+        httpd.serve_forever()
+
+    thread = threading.Thread(target=serve_httpd)
+    thread.start()
+
+    def stop_server() -> None:
+        httpd.shutdown()
+        httpd.server_close()
+        thread.join()
+
+    request.addfinalizer(stop_server)
+
+    return f"http://{addr}:{port}"

e2e_tests/test_standalone/test_study_list.py

@@ -0,0 +1,22 @@
+from playwright.sync_api import Page
+import pytest
+
+from ..test_server import make_standalone_server
+
+
+@pytest.fixture
+def server_url(request: pytest.FixtureRequest) -> str:
+    return make_standalone_server(request)
+
+
+def test_home(
+    page: Page,
+    server_url: str,
+) -> None:
+    url = f"{server_url}"
+    page.goto(url)
+    element = page.get_by_role("heading")
+    assert element is not None
+    title = element.text_content()
+    assert title is not None
+    assert title == "Optuna Dashboard (Wasm ver.)"