Merge pull request #85 from optuna/detect-crash

Run Visual Regression Test on GitHub Actions.

.github/workflows/integration-tests.yml

@@ -0,0 +1,51 @@
+name: integration-tests
+on:
+  pull_request:
+    branches:
+      - main
+    paths:
+      - '.github/workflows/integration-tests.yml'
+      - '**.py'
+      - '**.ts'
+      - '**.tsx'
+      - 'package.json'
+      - 'package-lock.json'
+      - 'tsconfig.json'
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Set up Node v14
+        uses: actions/setup-node@v2-beta
+        with:
+          node-version: '14'
+      - run: npm install
+      - run: npm run build:dev
+
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.x'
+          architecture: x64
+      - name: Install dependencies
+        run: |
+          python -m pip install --progress-bar off --upgrade pip setuptools
+          pip install --progress-bar off .
+          pip install --progress-bar off pyppeteer
+
+      - name: Cache headless chromium
+        id: cache-chromium
+        uses: actions/cache@v2
+        with:
+          path: ./local-chromium
+          key: chromium
+      - run: pyppeteer-install
+        if: steps.cache-chromium.outputs.cache-hit != 'true'
+        env:
+          PYPPETEER_HOME: ./local-chromium
+
+      - run: python visual_regression_test.py --skip-screenshot --sleep 3
+        env:
+          PYPPETEER_HOME: ./local-chromium

.github/workflows/typescript-checks.yml

@@ -6,6 +6,9 @@ on:
       - '.eslintrc.js'
       - '**.ts'
       - '**.tsx'
+      - 'package.json'
+      - 'package-lock.json'
+      - 'tsconfig.json'
 jobs:
   lint:
     name: Lint checking on Ubuntu

visual_regression_test.py

@@ -1,13 +1,16 @@
 import argparse
 import asyncio
 import os
+import sys
 import threading
 import time
+from typing import List
 from typing import Tuple
 from wsgiref.simple_server import make_server
 
 import optuna
 from pyppeteer import launch
+from pyppeteer.page import Page
 
 from optuna_dashboard.app import create_app
 
@@ -23,7 +26,7 @@ parser.add_argument(
     "--sleep",
     help="sleep seconds on each page open (default: %(default)s)",
     type=int,
-    default="10",
+    default=5,
 )
 parser.add_argument(
     "--output-dir", help="output directory (default: %(default)s)", default="tmp"
@@ -37,6 +40,9 @@ parser.add_argument(
 parser.add_argument(
     "--storage", help="storage url (default: %(default)s)", default=None
 )
+parser.add_argument(
+    "--skip-screenshot", help="skip to take screenshot", action="store_true"
+)
 args = parser.parse_args()
 
 
@@ -143,23 +149,45 @@ def create_dummy_storage() -> optuna.storages.InMemoryStorage:
     return storage
 
 
-async def take_screenshots(storage: optuna.storages.BaseStorage) -> None:
+async def contains_study_name(page: Page, study_name: str) -> bool:
+    h6_elements = await page.querySelectorAll("h6")
+    for element in h6_elements:
+        title = await page.evaluate("(element) => element.textContent", element)
+        if title == study_name:
+            return True
+    return False
+
+
+async def take_screenshots(storage: optuna.storages.BaseStorage) -> List[str]:
+    validation_errors: List[str] = []
+
     browser = await launch()
     page = await browser.newPage()
     await page.setViewport({"width": args.width, "height": args.height})
 
-    await page.goto(f"http://{args.host}:{args.port}/dashboard/")
-    time.sleep(1)
-    await page.screenshot({"path": os.path.join(args.output_dir, "study-list.png")})
+    if not args.skip_screenshot:
+        await page.goto(f"http://{args.host}:{args.port}/dashboard/")
+        time.sleep(1)
+        await page.screenshot({"path": os.path.join(args.output_dir, "study-list.png")})
 
     study_ids = {s._study_id: s.study_name for s in storage.get_all_study_summaries()}
     for study_id, study_name in study_ids.items():
         await page.goto(f"http://{args.host}:{args.port}/dashboard/studies/{study_id}")
         time.sleep(args.sleep)
-        await page.screenshot(
-            {"path": os.path.join(args.output_dir, f"study-{study_name}.png")}
-        )
+
+        if not args.skip_screenshot:
+            await page.screenshot(
+                {"path": os.path.join(args.output_dir, f"study-{study_name}.png")}
+            )
+
+        is_crashed = not await contains_study_name(page, study_name)
+        if is_crashed:
+            validation_errors.append(
+                f"Page is crashed at study_name='{study_name}' (id={study_id})"
+            )
+
     await browser.close()
+    return validation_errors
 
 
 def main() -> None:
@@ -179,12 +207,17 @@ def main() -> None:
     thread.start()
 
     loop = asyncio.get_event_loop()
-    loop.run_until_complete(take_screenshots(storage))
+    error_messages = loop.run_until_complete(take_screenshots(storage))
+    for msg in error_messages:
+        print(msg)
 
     httpd.shutdown()
     httpd.server_close()
     thread.join()
 
+    if error_messages:
+        sys.exit(1)
+
 
 if __name__ == "__main__":
     main()