From 920770e5fa4d06ac3653f9e5a41b332188d66593 Mon Sep 17 00:00:00 2001 From: c-bata Date: Sat, 14 Jan 2023 00:22:26 +0900 Subject: [PATCH] Implement FileUpload on MarkdownEditor --- optuna_dashboard/_bottle_util.py | 4 +++ optuna_dashboard/artifact/_backend.py | 20 +++++++---- optuna_dashboard/ts/action.ts | 33 ++++++++++++++++- optuna_dashboard/ts/apiClient.ts | 17 +++++++++ optuna_dashboard/ts/components/Note.tsx | 48 ++++++++++++++++++++++++- optuna_dashboard/ts/state.ts | 5 +++ optuna_dashboard/ts/types/index.d.ts | 7 ++++ 7 files changed, 125 insertions(+), 9 deletions(-) diff --git a/optuna_dashboard/_bottle_util.py b/optuna_dashboard/_bottle_util.py index 2ab57e59..9799cebe 100644 --- a/optuna_dashboard/_bottle_util.py +++ b/optuna_dashboard/_bottle_util.py @@ -13,6 +13,7 @@ from typing import TypeVar from typing import Union from bottle import BaseResponse +from bottle import HTTPError from bottle import response @@ -29,6 +30,9 @@ def json_api_view(view: BottleAPIView) -> BottleAPIView: response.content_type = "application/json" response_body = view(*args, **kwargs) return response_body + except HTTPError as e: + response.status = e.status_code + return json.dumps({"reason": str(e.body)}) except Exception as e: response.status = 500 response.content_type = "application/json" diff --git a/optuna_dashboard/artifact/_backend.py b/optuna_dashboard/artifact/_backend.py index fd282de6..fee3e535 100644 --- a/optuna_dashboard/artifact/_backend.py +++ b/optuna_dashboard/artifact/_backend.py @@ -1,11 +1,13 @@ from __future__ import annotations +import io import json import mimetypes import os.path from typing import TYPE_CHECKING import uuid +from bottle import BaseRequest from bottle import Bottle from bottle import request from bottle import response @@ -37,6 +39,9 @@ if TYPE_CHECKING: ARTIFACTS_ATTR_PREFIX = "dashboard:artifacts:" DEFAULT_MIME_TYPE = "application/octet-stream" +BaseRequest.MEMFILE_MAX = int( + os.environ.get("OPTUNA_DASHBOARD_MEMFILE_MAX", 1024 * 1024 * 128) +) # 128MB def register_artifact_route( @@ -56,22 +61,22 @@ def register_artifact_route( body = f.read() return body - @app.post("/api/artifacts//") + @app.post("/api/studies//artifacts") @json_api_view - def upload_artifact(trial_id: int) -> dict[str, Any]: + def upload_artifact_api(study_id: int) -> dict[str, Any]: if artifact_backend is None: response.status = 400 # Bad Request return {"reason": "Cannot access to the artifacts."} file = request.json.get("file") - if file is None: + trial_id = request.json.get("trial_id") + if file is None or trial_id is None: response.status = 400 return {"reason": "Please specify the 'file' key."} _, data = parse_data_uri(file) filename = request.json.get("filename", "") artifact_id = str(uuid.uuid4()) - with artifact_backend.open(artifact_id=artifact_id) as f: - f.write(data) + artifact_backend.write(artifact_id, io.BytesIO(data)) mimetype, encoding = mimetypes.guess_type(filename) artifact = { @@ -81,7 +86,7 @@ def register_artifact_route( "encoding": encoding, } attr_key = _artifact_prefix(trial_id=trial_id) + artifact_id - storage.set_study_system_attr(trial_id, attr_key, json.dumps(artifact)) + storage.set_study_system_attr(study_id, attr_key, json.dumps(artifact)) response.status = 201 return artifact @@ -124,6 +129,7 @@ def upload_artifact( filename = os.path.basename(file_path) storage = trial.storage trial_id = trial._trial_id + study_id = trial._study_id artifact_id = str(uuid.uuid4()) guess_mimetype, guess_encoding = mimetypes.guess_type(filename) artifact: ArtifactMeta = { @@ -133,7 +139,7 @@ def upload_artifact( "filename": filename, } attr_key = _artifact_prefix(trial_id=trial_id) + artifact_id - storage.set_study_system_attr(trial_id, attr_key, json.dumps(artifact)) + storage.set_study_system_attr(study_id, attr_key, json.dumps(artifact)) with open(file_path, "rb") as f: backend.write(artifact_id, f) diff --git a/optuna_dashboard/ts/action.ts b/optuna_dashboard/ts/action.ts index edec6741..0a28ae27 100644 --- a/optuna_dashboard/ts/action.ts +++ b/optuna_dashboard/ts/action.ts @@ -1,4 +1,4 @@ -import { useRecoilState } from "recoil" +import { useRecoilState, useSetRecoilState } from "recoil" import { useSnackbar } from "notistack" import { getStudyDetailAPI, @@ -9,12 +9,14 @@ import { saveStudyNoteAPI, saveTrialNoteAPI, renameStudyAPI, + uploadArtifactAPI, } from "./apiClient" import { graphVisibilityState, studyDetailsState, studySummariesState, paramImportanceState, + isFileUploading, } from "./state" const localStorageGraphVisibility = "graphVisibility" @@ -29,6 +31,7 @@ export const actionCreator = () => { useRecoilState(graphVisibilityState) const [paramImportance, setParamImportance] = useRecoilState(paramImportanceState) + const setUploading = useSetRecoilState(isFileUploading) const setStudyDetailState = (studyId: number, study: StudyDetail) => { const newVal = Object.assign({}, studyDetails) @@ -265,6 +268,33 @@ export const actionCreator = () => { }) } + const uploadArtifact = ( + studyId: number, + trialId: number, + file: File + ): void => { + const reader = new FileReader() + setUploading(true) + reader.readAsDataURL(file) + reader.onload = (upload: any) => { + uploadArtifactAPI(studyId, trialId, file.name, upload.target.result) + .then((artifact) => { + setUploading(false) + // TODO: update global state + console.dir(artifact) + }) + .catch((err) => { + setUploading(false) + const reason = err.response?.data.reason + enqueueSnackbar(`Failed to upload ${reason}`, { variant: "error" }) + }) + } + reader.onerror = (error) => { + enqueueSnackbar(`Failed to read the file ${error}`, { variant: "error" }) + console.log(error) + } + } + return { updateStudyDetail, updateStudySummaries, @@ -276,6 +306,7 @@ export const actionCreator = () => { saveGraphVisibility, saveStudyNote, saveTrialNote, + uploadArtifact, } } diff --git a/optuna_dashboard/ts/apiClient.ts b/optuna_dashboard/ts/apiClient.ts index bf4c1a6a..21102095 100644 --- a/optuna_dashboard/ts/apiClient.ts +++ b/optuna_dashboard/ts/apiClient.ts @@ -217,6 +217,23 @@ export const saveTrialNoteAPI = ( }) } +export const uploadArtifactAPI = ( + studyId: number, + trialId: number, + fileName: string, + dataUrl: string +): Promise => { + return axiosInstance + .post(`/api/studies/${studyId}/artifacts`, { + trial_id: trialId, + file: dataUrl, + filename: fileName, + }) + .then((res) => { + return res.data + }) +} + interface ParamImportancesResponse { param_importances: ParamImportance[][] } diff --git a/optuna_dashboard/ts/components/Note.tsx b/optuna_dashboard/ts/components/Note.tsx index e482aa47..ad81375e 100644 --- a/optuna_dashboard/ts/components/Note.tsx +++ b/optuna_dashboard/ts/components/Note.tsx @@ -15,7 +15,13 @@ import { Typography, useTheme, } from "@mui/material" -import React, { FC, createRef, useState, useEffect } from "react" +import React, { + FC, + createRef, + useState, + useEffect, + DragEventHandler, +} from "react" import ReactMarkdown from "react-markdown" import remarkGfm from "remark-gfm" import remarkMath from "remark-math" @@ -36,6 +42,8 @@ import { Prism as SyntaxHighlighter } from "react-syntax-highlighter" import { darcula } from "react-syntax-highlighter/dist/esm/styles/prism" import { actionCreator } from "../action" +import { useRecoilValue } from "recoil" +import { isFileUploading } from "../state" const placeholder = `## What is this feature for? @@ -171,9 +179,11 @@ const MarkdownEditorModal: FC<{ window.onbeforeunload = null }) + const [dragOver, setDragOver] = useState(false) const [saving, setSaving] = useState(false) const [edited, setEdited] = useState(false) const [curNote, setCurNote] = useState({ version: 0, body: "" }) + const uploading = useRecoilValue(isFileUploading) const textAreaRef = createRef() const notLatest = latestNote.version > curNote.version @@ -225,6 +235,37 @@ const MarkdownEditorModal: FC<{ window.onbeforeunload = null } + const handleDrop: DragEventHandler = (e) => { + if (trialId === undefined) { + return + } + e.stopPropagation() + e.preventDefault() + const file = e.dataTransfer.files[0] + setDragOver(false) + action.uploadArtifact(studyId, trialId, file) + } + + const handleDragOver: DragEventHandler = (e) => { + if (trialId === undefined) { + return + } + e.stopPropagation() + e.preventDefault() + e.dataTransfer.dropEffect = "copy" + setDragOver(true) + } + + const handleDragLeave: DragEventHandler = (e) => { + if (trialId === undefined) { + return + } + e.stopPropagation() + e.preventDefault() + e.dataTransfer.dropEffect = "copy" + setDragOver(false) + } + // See https://github.com/iamhosseindhv/notistack/issues/231#issuecomment-825924840 const zIndex = theme.zIndex.snackbar - 2 @@ -273,10 +314,15 @@ const MarkdownEditorModal: FC<{ > + {dragOver ? "DragOver=true" : "DragOver=false"} + {uploading ? "uploading=true" : "not uploading"} ({ default: false, }) +export const isFileUploading = atom({ + key: "isFileUploading", + default: false, +}) + export const useStudyDetailValue = (studyId: number): StudyDetail | null => { const studyDetails = useRecoilValue(studyDetailsState) return studyDetails[studyId] || null diff --git a/optuna_dashboard/ts/types/index.d.ts b/optuna_dashboard/ts/types/index.d.ts index c58928cd..a7a93e00 100644 --- a/optuna_dashboard/ts/types/index.d.ts +++ b/optuna_dashboard/ts/types/index.d.ts @@ -88,6 +88,13 @@ type Note = { body: string } +type Artifact = { + artifact_id: string + filename: string + mimetype: string + encoding: string +} + type Trial = { trial_id: number study_id: number