diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml new file mode 100644 index 00000000..e8a688c1 --- /dev/null +++ b/.github/workflows/integration-tests.yml @@ -0,0 +1,51 @@ +name: integration-tests +on: + pull_request: + branches: + - main + paths: + - '.github/workflows/integration-tests.yml' + - '**.py' + - '**.ts' + - '**.tsx' + - 'package.json' + - 'package-lock.json' + - 'tsconfig.json' +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - name: Set up Node v14 + uses: actions/setup-node@v2-beta + with: + node-version: '14' + - run: npm install + - run: npm run build:dev + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.x' + architecture: x64 + - name: Install dependencies + run: | + python -m pip install --progress-bar off --upgrade pip setuptools + pip install --progress-bar off . + pip install --progress-bar off pyppeteer + + - name: Cache headless chromium + id: cache-chromium + uses: actions/cache@v2 + with: + path: ./local-chromium + key: chromium + - run: pyppeteer-install + if: steps.cache-chromium.outputs.cache-hit != 'true' + env: + PYPPETEER_HOME: ./local-chromium + + - run: python visual_regression_test.py --skip-screenshot --sleep 3 + env: + PYPPETEER_HOME: ./local-chromium diff --git a/.github/workflows/typescript-checks.yml b/.github/workflows/typescript-checks.yml index c2317dae..525e1d6b 100644 --- a/.github/workflows/typescript-checks.yml +++ b/.github/workflows/typescript-checks.yml @@ -6,6 +6,9 @@ on: - '.eslintrc.js' - '**.ts' - '**.tsx' + - 'package.json' + - 'package-lock.json' + - 'tsconfig.json' jobs: lint: name: Lint checking on Ubuntu diff --git a/visual_regression_test.py b/visual_regression_test.py index 63632925..392e179d 100644 --- a/visual_regression_test.py +++ b/visual_regression_test.py @@ -1,13 +1,16 @@ import argparse import asyncio import os +import sys import threading import time +from typing import List from typing import Tuple from wsgiref.simple_server import make_server import optuna from pyppeteer import launch +from pyppeteer.page import Page from optuna_dashboard.app import create_app @@ -23,7 +26,7 @@ parser.add_argument( "--sleep", help="sleep seconds on each page open (default: %(default)s)", type=int, - default="10", + default=5, ) parser.add_argument( "--output-dir", help="output directory (default: %(default)s)", default="tmp" @@ -37,6 +40,9 @@ parser.add_argument( parser.add_argument( "--storage", help="storage url (default: %(default)s)", default=None ) +parser.add_argument( + "--skip-screenshot", help="skip to take screenshot", action="store_true" +) args = parser.parse_args() @@ -143,23 +149,45 @@ def create_dummy_storage() -> optuna.storages.InMemoryStorage: return storage -async def take_screenshots(storage: optuna.storages.BaseStorage) -> None: +async def contains_study_name(page: Page, study_name: str) -> bool: + h6_elements = await page.querySelectorAll("h6") + for element in h6_elements: + title = await page.evaluate("(element) => element.textContent", element) + if title == study_name: + return True + return False + + +async def take_screenshots(storage: optuna.storages.BaseStorage) -> List[str]: + validation_errors: List[str] = [] + browser = await launch() page = await browser.newPage() await page.setViewport({"width": args.width, "height": args.height}) - await page.goto(f"http://{args.host}:{args.port}/dashboard/") - time.sleep(1) - await page.screenshot({"path": os.path.join(args.output_dir, "study-list.png")}) + if not args.skip_screenshot: + await page.goto(f"http://{args.host}:{args.port}/dashboard/") + time.sleep(1) + await page.screenshot({"path": os.path.join(args.output_dir, "study-list.png")}) study_ids = {s._study_id: s.study_name for s in storage.get_all_study_summaries()} for study_id, study_name in study_ids.items(): await page.goto(f"http://{args.host}:{args.port}/dashboard/studies/{study_id}") time.sleep(args.sleep) - await page.screenshot( - {"path": os.path.join(args.output_dir, f"study-{study_name}.png")} - ) + + if not args.skip_screenshot: + await page.screenshot( + {"path": os.path.join(args.output_dir, f"study-{study_name}.png")} + ) + + is_crashed = not await contains_study_name(page, study_name) + if is_crashed: + validation_errors.append( + f"Page is crashed at study_name='{study_name}' (id={study_id})" + ) + await browser.close() + return validation_errors def main() -> None: @@ -179,12 +207,17 @@ def main() -> None: thread.start() loop = asyncio.get_event_loop() - loop.run_until_complete(take_screenshots(storage)) + error_messages = loop.run_until_complete(take_screenshots(storage)) + for msg in error_messages: + print(msg) httpd.shutdown() httpd.server_close() thread.join() + if error_messages: + sys.exit(1) + if __name__ == "__main__": main()