From 21914fbdbd85ffea8d4d21d92348e41e3335eafc Mon Sep 17 00:00:00 2001 From: c-bata Date: Sun, 11 Apr 2021 18:12:44 +0900 Subject: [PATCH 1/4] Detect crash on visual_regression_test --- visual_regression_test.py | 44 +++++++++++++++++++++++++++++++++------ 1 file changed, 38 insertions(+), 6 deletions(-) diff --git a/visual_regression_test.py b/visual_regression_test.py index 63632925..600cf21e 100644 --- a/visual_regression_test.py +++ b/visual_regression_test.py @@ -1,13 +1,16 @@ import argparse import asyncio import os +import sys import threading import time +from typing import List from typing import Tuple from wsgiref.simple_server import make_server import optuna from pyppeteer import launch +from pyppeteer.page import Page from optuna_dashboard.app import create_app @@ -23,7 +26,7 @@ parser.add_argument( "--sleep", help="sleep seconds on each page open (default: %(default)s)", type=int, - default="10", + default=5, ) parser.add_argument( "--output-dir", help="output directory (default: %(default)s)", default="tmp" @@ -37,6 +40,9 @@ parser.add_argument( parser.add_argument( "--storage", help="storage url (default: %(default)s)", default=None ) +parser.add_argument( + "--skip-screenshot", help="skip to take screenshot", action="store_true" +) args = parser.parse_args() @@ -143,7 +149,18 @@ def create_dummy_storage() -> optuna.storages.InMemoryStorage: return storage -async def take_screenshots(storage: optuna.storages.BaseStorage) -> None: +async def contains_study_name(page: Page, study_name: str) -> bool: + h6_elements = await page.querySelectorAll("h6") + for element in h6_elements: + title = await page.evaluate("(element) => element.textContent", element) + if title == study_name: + return True + return False + + +async def take_screenshots(storage: optuna.storages.BaseStorage) -> List[str]: + validation_errors: List[str] = [] + browser = await launch() page = await browser.newPage() await page.setViewport({"width": args.width, "height": args.height}) @@ -156,10 +173,20 @@ async def take_screenshots(storage: optuna.storages.BaseStorage) -> None: for study_id, study_name in study_ids.items(): await page.goto(f"http://{args.host}:{args.port}/dashboard/studies/{study_id}") time.sleep(args.sleep) - await page.screenshot( - {"path": os.path.join(args.output_dir, f"study-{study_name}.png")} - ) + + if not args.skip_screenshot: + await page.screenshot( + {"path": os.path.join(args.output_dir, f"study-{study_name}.png")} + ) + + is_crashed = not await contains_study_name(page, study_name) + if is_crashed: + validation_errors.append( + f"Page is crashed at study_name='{study_name}' (id={study_id})" + ) + await browser.close() + return validation_errors def main() -> None: @@ -179,12 +206,17 @@ def main() -> None: thread.start() loop = asyncio.get_event_loop() - loop.run_until_complete(take_screenshots(storage)) + error_messages = loop.run_until_complete(take_screenshots(storage)) + for msg in error_messages: + print(msg) httpd.shutdown() httpd.server_close() thread.join() + if error_messages: + sys.exit(1) + if __name__ == "__main__": main() From 24d24cf84efd3da3d42b0f94c0de8a64f0e31806 Mon Sep 17 00:00:00 2001 From: c-bata Date: Sun, 11 Apr 2021 18:39:53 +0900 Subject: [PATCH 2/4] Run integration tests on GitHub Actions --- .github/workflows/integration-tests.yml | 51 +++++++++++++++++++++++++ .github/workflows/typescript-checks.yml | 3 ++ 2 files changed, 54 insertions(+) create mode 100644 .github/workflows/integration-tests.yml diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml new file mode 100644 index 00000000..b1a3e0a2 --- /dev/null +++ b/.github/workflows/integration-tests.yml @@ -0,0 +1,51 @@ +name: integration-tests +on: + pull_request: + branches: + - main + paths: + - '.github/workflows/integration-tests.yml' + - '**.py' + - '**.ts' + - '**.tsx' + - 'package.json' + - 'package-lock.json' + - 'tsconfig.json' +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - name: Set up Node v14 + uses: actions/setup-node@v2-beta + with: + node-version: '14' + - run: npm install + - run: npm run build:dev + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.x' + architecture: x64 + - name: Install dependencies + run: | + python -m pip install --progress-bar off --upgrade pip setuptools + pip install --progress-bar off . + pip install --progress-bar off pyppeteer + + - name: Cache headless chromium + id: cache-chromium + uses: actions/cache@v2 + with: + path: ./local-chromium + key: chromium + - run: pyppeteer-install + if: steps.cache-chromium.outputs.cache-hit != 'true' + env: + PYPPETEER_HOME: ./local-chromium + + - run: python visual_regression_test.py + env: + PYPPETEER_HOME: ./local-chromium --no-screenshot --sleep 3 diff --git a/.github/workflows/typescript-checks.yml b/.github/workflows/typescript-checks.yml index c2317dae..525e1d6b 100644 --- a/.github/workflows/typescript-checks.yml +++ b/.github/workflows/typescript-checks.yml @@ -6,6 +6,9 @@ on: - '.eslintrc.js' - '**.ts' - '**.tsx' + - 'package.json' + - 'package-lock.json' + - 'tsconfig.json' jobs: lint: name: Lint checking on Ubuntu From c378a1a60a7a3be58ffeee35e31e4825233ff3c7 Mon Sep 17 00:00:00 2001 From: c-bata Date: Sun, 11 Apr 2021 18:43:51 +0900 Subject: [PATCH 3/4] Fix integration tests --- .github/workflows/integration-tests.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index b1a3e0a2..4066a408 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -12,7 +12,7 @@ on: - 'package-lock.json' - 'tsconfig.json' jobs: - lint: + test: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 @@ -46,6 +46,6 @@ jobs: env: PYPPETEER_HOME: ./local-chromium - - run: python visual_regression_test.py + - run: python visual_regression_test.py --no-screenshot --sleep 3 env: - PYPPETEER_HOME: ./local-chromium --no-screenshot --sleep 3 + PYPPETEER_HOME: ./local-chromium From a07377be0fa8fbd60d404d5c19e836296f6a4255 Mon Sep 17 00:00:00 2001 From: c-bata Date: Sun, 11 Apr 2021 18:46:57 +0900 Subject: [PATCH 4/4] Fix integration tests --- .github/workflows/integration-tests.yml | 2 +- visual_regression_test.py | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 4066a408..e8a688c1 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -46,6 +46,6 @@ jobs: env: PYPPETEER_HOME: ./local-chromium - - run: python visual_regression_test.py --no-screenshot --sleep 3 + - run: python visual_regression_test.py --skip-screenshot --sleep 3 env: PYPPETEER_HOME: ./local-chromium diff --git a/visual_regression_test.py b/visual_regression_test.py index 600cf21e..392e179d 100644 --- a/visual_regression_test.py +++ b/visual_regression_test.py @@ -165,9 +165,10 @@ async def take_screenshots(storage: optuna.storages.BaseStorage) -> List[str]: page = await browser.newPage() await page.setViewport({"width": args.width, "height": args.height}) - await page.goto(f"http://{args.host}:{args.port}/dashboard/") - time.sleep(1) - await page.screenshot({"path": os.path.join(args.output_dir, "study-list.png")}) + if not args.skip_screenshot: + await page.goto(f"http://{args.host}:{args.port}/dashboard/") + time.sleep(1) + await page.screenshot({"path": os.path.join(args.output_dir, "study-list.png")}) study_ids = {s._study_id: s.study_name for s in storage.get_all_study_summaries()} for study_id, study_name in study_ids.items():