Merge pull request #674 from gen740/add_upload_study_artifact_api

Add upload study artifact api
This commit is contained in:
keisuke umezawa
2023-11-23 18:26:56 +09:00
committed by GitHub
9 changed files with 511 additions and 53 deletions
+54 -6
View File
@@ -105,7 +105,7 @@ def register_artifact_route(
@app.post("/api/artifacts/<study_id:int>/<trial_id:int>")
@json_api_view
def upload_artifact_api(study_id: int, trial_id: int) -> dict[str, Any]:
def upload_trial_artifact_api(study_id: int, trial_id: int) -> dict[str, Any]:
trial = storage.get_trial(trial_id)
if trial is None:
response.status = 400
@@ -144,9 +144,42 @@ def register_artifact_route(
"artifacts": list_trial_artifacts(storage.get_study_system_attrs(study_id), trial),
}
@app.post("/api/artifacts/<study_id:int>")
@json_api_view
def upload_study_artifact_api(study_id: int) -> dict[str, Any]:
if artifact_store is None:
response.status = 400 # Bad Request
return {"reason": "Cannot access to the artifacts."}
file = request.json.get("file")
if file is None:
response.status = 400
return {"reason": "Please specify the 'file' key."}
_, data = parse_data_uri(file)
filename = request.json.get("filename", "")
artifact_id = str(uuid.uuid4())
artifact_store.write(artifact_id, io.BytesIO(data))
mimetype, encoding = mimetypes.guess_type(filename)
artifact = {
"artifact_id": artifact_id,
"filename": filename,
"mimetype": mimetype or DEFAULT_MIME_TYPE,
"encoding": encoding,
}
attr_key = ARTIFACTS_ATTR_PREFIX + artifact_id
storage.set_study_system_attr(study_id, attr_key, json.dumps(artifact))
response.status = 201
return {
"artifact_id": artifact_id,
"artifacts": list_study_artifacts(storage.get_study_system_attrs(study_id)),
}
@app.delete("/api/artifacts/<study_id:int>/<trial_id:int>/<artifact_id:re:[0-9a-fA-F-]+>")
@json_api_view
def delete_artifact(study_id: int, trial_id: int, artifact_id: str) -> dict[str, Any]:
def delete_trial_artifact(study_id: int, trial_id: int, artifact_id: str) -> dict[str, Any]:
if artifact_store is None:
response.status = 400 # Bad Request
return {"reason": "Cannot access to the artifacts."}
@@ -154,7 +187,7 @@ def register_artifact_route(
# The artifact's metadata is stored in one of the following two locations:
storage.set_study_system_attr(
study_id, _dashboard_trial_artifact_prefix(trial_id) + artifact_id, json.dumps(None)
study_id, _dashboard_artifact_prefix(trial_id) + artifact_id, json.dumps(None)
)
storage.set_trial_system_attr(
trial_id, ARTIFACTS_ATTR_PREFIX + artifact_id, json.dumps(None)
@@ -163,6 +196,21 @@ def register_artifact_route(
response.status = 204
return {}
@app.delete("/api/artifacts/<study_id:int>/<artifact_id:re:[0-9a-fA-F-]+>")
@json_api_view
def delete_study_artifact(study_id: int, artifact_id: str) -> dict[str, Any]:
if artifact_store is None:
response.status = 400 # Bad Request
return {"reason": "Cannot access to the artifacts."}
artifact_store.remove(artifact_id)
storage.set_study_system_attr(
study_id, ARTIFACTS_ATTR_PREFIX + artifact_id, json.dumps(None)
)
response.status = 204
return {}
def upload_artifact(
backend: ArtifactBackend,
@@ -220,7 +268,7 @@ def upload_artifact(
return artifact_id
def _dashboard_trial_artifact_prefix(trial_id: int) -> str:
def _dashboard_artifact_prefix(trial_id: int) -> str:
return DASHBOARD_ARTIFACTS_ATTR_PREFIX + f"{trial_id}:"
@@ -240,7 +288,7 @@ def get_trial_artifact_meta(
) -> Optional[ArtifactMeta]:
# Search study_system_attrs due to backward compatibility.
study_system_attrs = storage.get_study_system_attrs(study_id)
attr_key = _dashboard_trial_artifact_prefix(trial_id=trial_id) + artifact_id
attr_key = _dashboard_artifact_prefix(trial_id=trial_id) + artifact_id
artifact_meta = study_system_attrs.get(attr_key)
if artifact_meta is not None:
return json.loads(artifact_meta)
@@ -284,7 +332,7 @@ def list_trial_artifacts(
dashboard_artifact_metas = [
json.loads(value)
for key, value in study_system_attrs.items()
if key.startswith(_dashboard_trial_artifact_prefix(trial._trial_id))
if key.startswith(_dashboard_artifact_prefix(trial._trial_id))
]
# Collect ArtifactMeta from trial_system_attrs. Note that artifacts uploaded via
+74 -10
View File
@@ -11,9 +11,11 @@ import {
tellTrialAPI,
saveTrialUserAttrsAPI,
renameStudyAPI,
uploadArtifactAPI,
uploadTrialArtifactAPI,
uploadStudyArtifactAPI,
getMetaInfoAPI,
deleteArtifactAPI,
deleteTrialArtifactAPI,
deleteStudyArtifactAPI,
reportPreferenceAPI,
skipPreferentialTrialAPI,
removePreferentialHistoryAPI,
@@ -100,7 +102,13 @@ export const actionCreator = () => {
setTrial(studyId, trialIndex, newTrial)
}
const deleteTrialArtifact = (
const setStudyArtifacts = (studyId: number, artifacts: Artifact[]) => {
const newStudy: StudyDetail = Object.assign({}, studyDetails[studyId])
newStudy.artifacts = artifacts
setStudyDetailState(studyId, newStudy)
}
const deleteTrialArtifactState = (
studyId: number,
trialId: number,
artifact_id: string
@@ -122,6 +130,18 @@ export const actionCreator = () => {
setTrialArtifacts(studyId, index, newArtifacts)
}
const deleteStudyArtifactState = (studyId: number, artifact_id: string) => {
const artifacts = studyDetails[studyId].artifacts
const artifactIndex = artifacts.findIndex(
(a) => a.artifact_id === artifact_id
)
const newArtifacts = [
...artifacts.slice(0, artifactIndex),
...artifacts.slice(artifactIndex + 1, artifacts.length),
]
setStudyArtifacts(studyId, newArtifacts)
}
const setTrialStateValues = (
studyId: number,
index: number,
@@ -430,7 +450,7 @@ export const actionCreator = () => {
})
}
const uploadArtifact = (
const uploadTrialArtifact = (
studyId: number,
trialId: number,
file: File
@@ -439,7 +459,7 @@ export const actionCreator = () => {
setUploading(true)
reader.readAsDataURL(file)
reader.onload = (upload: ProgressEvent<FileReader>) => {
uploadArtifactAPI(
uploadTrialArtifactAPI(
studyId,
trialId,
file.name,
@@ -467,14 +487,56 @@ export const actionCreator = () => {
}
}
const deleteArtifact = (
const uploadStudyArtifact = (studyId: number, file: File): void => {
const reader = new FileReader()
setUploading(true)
reader.readAsDataURL(file)
reader.onload = (upload: ProgressEvent<FileReader>) => {
uploadStudyArtifactAPI(
studyId,
file.name,
upload.target?.result as string
)
.then((res) => {
setUploading(false)
setStudyArtifacts(studyId, res.artifacts)
})
.catch((err) => {
setUploading(false)
const reason = err.response?.data.reason
enqueueSnackbar(`Failed to upload ${reason}`, { variant: "error" })
})
}
reader.onerror = (error) => {
enqueueSnackbar(`Failed to read the file ${error}`, { variant: "error" })
console.log(error)
}
}
const deleteTrialArtifact = (
studyId: number,
trialId: number,
artifactId: string
): void => {
deleteArtifactAPI(studyId, trialId, artifactId)
deleteTrialArtifactAPI(studyId, trialId, artifactId)
.then(() => {
deleteTrialArtifact(studyId, trialId, artifactId)
deleteTrialArtifactState(studyId, trialId, artifactId)
enqueueSnackbar(`Success to delete an artifact.`, {
variant: "success",
})
})
.catch((err) => {
const reason = err.response?.data.reason
enqueueSnackbar(`Failed to delete ${reason}.`, {
variant: "error",
})
})
}
const deleteStudyArtifact = (studyId: number, artifactId: string): void => {
deleteStudyArtifactAPI(studyId, artifactId)
.then(() => {
deleteStudyArtifactState(studyId, artifactId)
enqueueSnackbar(`Success to delete an artifact.`, {
variant: "success",
})
@@ -693,8 +755,10 @@ export const actionCreator = () => {
saveReloadInterval,
saveStudyNote,
saveTrialNote,
uploadArtifact,
deleteArtifact,
uploadTrialArtifact,
uploadStudyArtifact,
deleteTrialArtifact,
deleteStudyArtifact,
makeTrialComplete,
makeTrialFail,
saveTrialUserAttrs,
+28 -2
View File
@@ -280,7 +280,7 @@ type UploadArtifactAPIResponse = {
artifacts: Artifact[]
}
export const uploadArtifactAPI = (
export const uploadTrialArtifactAPI = (
studyId: number,
trialId: number,
fileName: string,
@@ -296,7 +296,22 @@ export const uploadArtifactAPI = (
})
}
export const deleteArtifactAPI = (
export const uploadStudyArtifactAPI = (
studyId: number,
fileName: string,
dataUrl: string
): Promise<UploadArtifactAPIResponse> => {
return axiosInstance
.post<UploadArtifactAPIResponse>(`/api/artifacts/${studyId}`, {
file: dataUrl,
filename: fileName,
})
.then((res) => {
return res.data
})
}
export const deleteTrialArtifactAPI = (
studyId: number,
trialId: number,
artifactId: string
@@ -308,6 +323,17 @@ export const deleteArtifactAPI = (
})
}
export const deleteStudyArtifactAPI = (
studyId: number,
artifactId: string
): Promise<void> => {
return axiosInstance
.delete<void>(`/api/artifacts/${studyId}/${artifactId}`)
.then(() => {
return
})
}
export const tellTrialAPI = (
trialId: number,
state: TrialStateFinished,
@@ -1,4 +1,4 @@
import React, { ReactNode, useState } from "react"
import React, { ReactNode, useState, FC } from "react"
import {
Dialog,
DialogTitle,
@@ -9,7 +9,7 @@ import {
} from "@mui/material"
import { actionCreator } from "../action"
export const useDeleteArtifactDialog = (): [
export const useDeleteTrialArtifactDialog = (): [
(studyId: number, trialId: number, artifact: Artifact) => void,
() => ReactNode
] => {
@@ -33,7 +33,7 @@ export const useDeleteArtifactDialog = (): [
if (artifact === null) {
return
}
action.deleteArtifact(studyId, trialId, artifact.artifact_id)
action.deleteTrialArtifact(studyId, trialId, artifact.artifact_id)
setOpenDeleteArtifactDialog(false)
setTarget([-1, -1, null])
}
@@ -45,32 +45,96 @@ export const useDeleteArtifactDialog = (): [
const renderDeleteArtifactDialog = () => {
return (
<Dialog
open={openDeleteArtifactDialog}
onClose={() => {
handleCloseDeleteArtifactDialog()
}}
aria-labelledby="delete-artifact-dialog-title"
>
<DialogTitle id="delete-artifact-dialog-title">
Delete artifact
</DialogTitle>
<DialogContent>
<DialogContentText>
Are you sure you want to delete an artifact ("
{target[2]?.filename}")?
</DialogContentText>
</DialogContent>
<DialogActions>
<Button onClick={handleCloseDeleteArtifactDialog} color="primary">
No
</Button>
<Button onClick={handleDeleteArtifact} color="primary">
Yes
</Button>
</DialogActions>
</Dialog>
<DeleteDialog
openDeleteArtifactDialog={openDeleteArtifactDialog}
handleCloseDeleteArtifactDialog={handleCloseDeleteArtifactDialog}
filename={target[2]?.filename}
handleDeleteArtifact={handleDeleteArtifact}
/>
)
}
return [openDialog, renderDeleteArtifactDialog]
}
export const useDeleteStudyArtifactDialog = (): [
(studyId: number, artifact: Artifact) => void,
() => ReactNode
] => {
const action = actionCreator()
const [openDeleteArtifactDialog, setOpenDeleteArtifactDialog] =
useState(false)
const [target, setTarget] = useState<[number, Artifact | null]>([-1, null])
const handleCloseDeleteArtifactDialog = () => {
setOpenDeleteArtifactDialog(false)
setTarget([-1, null])
}
const handleDeleteArtifact = () => {
const [studyId, artifact] = target
if (artifact === null) {
return
}
action.deleteStudyArtifact(studyId, artifact.artifact_id)
setOpenDeleteArtifactDialog(false)
setTarget([-1, null])
}
const openDialog = (studyId: number, artifact: Artifact) => {
setTarget([studyId, artifact])
setOpenDeleteArtifactDialog(true)
}
const renderDeleteArtifactDialog = () => {
return (
<DeleteDialog
openDeleteArtifactDialog={openDeleteArtifactDialog}
handleCloseDeleteArtifactDialog={handleCloseDeleteArtifactDialog}
filename={target[1]?.filename}
handleDeleteArtifact={handleDeleteArtifact}
/>
)
}
return [openDialog, renderDeleteArtifactDialog]
}
const DeleteDialog: FC<{
openDeleteArtifactDialog: boolean
handleCloseDeleteArtifactDialog: () => void
filename: string | undefined
handleDeleteArtifact: () => void
}> = ({
openDeleteArtifactDialog,
handleCloseDeleteArtifactDialog,
filename,
handleDeleteArtifact,
}) => {
return (
<Dialog
open={openDeleteArtifactDialog}
onClose={() => {
handleCloseDeleteArtifactDialog()
}}
aria-labelledby="delete-artifact-dialog-title"
>
<DialogTitle id="delete-artifact-dialog-title">
Delete artifact
</DialogTitle>
<DialogContent>
<DialogContentText>
Are you sure you want to delete an artifact ("
{filename}")?
</DialogContentText>
</DialogContent>
<DialogActions>
<Button onClick={handleCloseDeleteArtifactDialog} color="primary">
No
</Button>
<Button onClick={handleDeleteArtifact} color="primary">
Yes
</Button>
</DialogActions>
</Dialog>
)
}
+2 -2
View File
@@ -425,7 +425,7 @@ const ArtifactUploader: FC<{
if (files === null) {
return
}
action.uploadArtifact(studyId, trialId, files[0])
action.uploadTrialArtifact(studyId, trialId, files[0])
}
const handleDrop: DragEventHandler = (e) => {
@@ -433,7 +433,7 @@ const ArtifactUploader: FC<{
e.preventDefault()
const file = e.dataTransfer.files[0]
setDragOver(false)
action.uploadArtifact(studyId, trialId, file)
action.uploadTrialArtifact(studyId, trialId, file)
}
const handleDragOver: DragEventHandler = (e) => {
@@ -0,0 +1,226 @@
import React, {
ChangeEventHandler,
DragEventHandler,
FC,
MouseEventHandler,
useRef,
useState,
} from "react"
import {
Typography,
Box,
useTheme,
IconButton,
Card,
CardContent,
CardActionArea,
} from "@mui/material"
import UploadFileIcon from "@mui/icons-material/UploadFile"
import DownloadIcon from "@mui/icons-material/Download"
import DeleteIcon from "@mui/icons-material/Delete"
import FullscreenIcon from "@mui/icons-material/Fullscreen"
import { actionCreator } from "../action"
import { useDeleteStudyArtifactDialog } from "./DeleteArtifactDialog"
import {
useThreejsArtifactModal,
isThreejsArtifact,
} from "./ThreejsArtifactViewer"
import { ArtifactCardMedia } from "./ArtifactCardMedia"
export const StudyArtifactCards: FC<{ study: StudyDetail }> = ({ study }) => {
const theme = useTheme()
const [openDeleteArtifactDialog, renderDeleteArtifactDialog] =
useDeleteStudyArtifactDialog()
const [openThreejsArtifactModal, renderThreejsArtifactModal] =
useThreejsArtifactModal()
const width = "200px"
const height = "150px"
return (
<>
<Box sx={{ display: "flex", flexWrap: "wrap", p: theme.spacing(1, 0) }}>
{study.artifacts.map((artifact) => {
const urlPath = `/artifacts/${study.id}/${artifact.artifact_id}`
return (
<Card
key={artifact.artifact_id}
sx={{
marginBottom: theme.spacing(2),
width: width,
margin: theme.spacing(0, 1, 1, 0),
border: `1px solid ${theme.palette.divider}`,
}}
>
<ArtifactCardMedia
artifact={artifact}
urlPath={urlPath}
height={height}
/>
<CardContent
sx={{
display: "flex",
flexDirection: "row",
padding: `${theme.spacing(1)} !important`,
}}
>
<Typography
sx={{
p: theme.spacing(0.5, 0),
flexGrow: 1,
wordWrap: "break-word",
maxWidth: `calc(100% - ${
isThreejsArtifact(artifact)
? theme.spacing(12)
: theme.spacing(8)
})`,
}}
>
{artifact.filename}
</Typography>
{isThreejsArtifact(artifact) ? (
<IconButton
aria-label="show artifact 3d model"
size="small"
color="inherit"
sx={{ margin: "auto 0" }}
onClick={() => {
openThreejsArtifactModal(urlPath, artifact)
}}
>
<FullscreenIcon />
</IconButton>
) : null}
<IconButton
aria-label="delete artifact"
size="small"
color="inherit"
sx={{ margin: "auto 0" }}
onClick={() => {
openDeleteArtifactDialog(study.id, artifact)
}}
>
<DeleteIcon />
</IconButton>
<IconButton
aria-label="download artifact"
size="small"
color="inherit"
download={artifact.filename}
sx={{ margin: "auto 0" }}
href={urlPath}
>
<DownloadIcon />
</IconButton>
</CardContent>
</Card>
)
})}
<StudyArtifactUploader study={study} width={width} height={height} />
</Box>
{renderDeleteArtifactDialog()}
{renderThreejsArtifactModal()}
</>
)
}
const StudyArtifactUploader: FC<{
study: StudyDetail
width: string
height: string
}> = ({ study, width, height }) => {
const theme = useTheme()
const [dragOver, setDragOver] = useState<boolean>(false)
const action = actionCreator()
const inputRef = useRef<HTMLInputElement>(null)
const handleClick: MouseEventHandler = () => {
if (!inputRef || !inputRef.current) {
return
}
inputRef.current.click()
}
const handleOnChange: ChangeEventHandler<HTMLInputElement> = (e) => {
const files = e.target.files
if (files === null) {
return
}
action.uploadStudyArtifact(study.id, files[0])
}
const handleDragOver: DragEventHandler = (e) => {
e.stopPropagation()
e.preventDefault()
e.dataTransfer.dropEffect = "copy"
setDragOver(true)
}
const handleDragLeave: DragEventHandler = (e) => {
e.stopPropagation()
e.preventDefault()
e.dataTransfer.dropEffect = "copy"
setDragOver(false)
}
const handleDrop: DragEventHandler = (e) => {
e.stopPropagation()
e.preventDefault()
const files = e.dataTransfer.files
setDragOver(false)
for (let i = 0; i < files.length; i++) {
action.uploadStudyArtifact(study.id, files[i])
}
}
return (
<Card
sx={{
marginBottom: theme.spacing(2),
width: width,
minHeight: height,
margin: theme.spacing(0, 1, 1, 0),
border: dragOver
? `3px dashed ${theme.palette.mode === "dark" ? "white" : "black"}`
: `1px solid ${theme.palette.divider}`,
}}
onDragOver={handleDragOver}
onDragLeave={handleDragLeave}
onDrop={handleDrop}
>
<CardActionArea
onClick={handleClick}
sx={{
height: "100%",
}}
>
<CardContent
sx={{
display: "flex",
height: "100%",
flexDirection: "column",
justifyContent: "center",
alignItems: "center",
}}
>
<UploadFileIcon
sx={{ fontSize: 80, marginBottom: theme.spacing(2) }}
/>
<input
type="file"
ref={inputRef}
onChange={handleOnChange}
style={{ display: "none" }}
/>
<Typography>Upload a New File</Typography>
<Typography
sx={{ textAlign: "center", color: theme.palette.grey.A400 }}
>
Drag your file here or click to browse.
</Typography>
</CardContent>
</CardActionArea>
</Card>
)
}
@@ -17,12 +17,15 @@ import { DataGrid, DataGridColumn } from "./DataGrid"
import { GraphHyperparameterImportance } from "./GraphHyperparameterImportances"
import { UserDefinedPlot } from "./UserDefinedPlot"
import { BestTrialsCard } from "./BestTrialsCard"
import { StudyArtifactCards } from "./StudyArtifactCards"
import { useRecoilValue } from "recoil"
import {
useStudyDetailValue,
useStudyDirections,
useStudySummaryValue,
} from "../state"
import FormControlLabel from "@mui/material/FormControlLabel"
import { artifactIsAvailable } from "../state"
export const StudyHistory: FC<{ studyId: number }> = ({ studyId }) => {
const theme = useTheme()
@@ -31,6 +34,7 @@ export const StudyHistory: FC<{ studyId: number }> = ({ studyId }) => {
const studyDetail = useStudyDetailValue(studyId)
const [logScale, setLogScale] = useState<boolean>(false)
const [includePruned, setIncludePruned] = useState<boolean>(true)
const artifactEnabled = useRecoilValue<boolean>(artifactIsAvailable)
const handleLogScaleChange = () => {
setLogScale(!logScale)
@@ -167,6 +171,32 @@ export const StudyHistory: FC<{ studyId: number }> = ({ studyId }) => {
</Card>
</Grid2>
</Grid2>
{artifactEnabled && studyDetail !== null && (
<Grid2 container spacing={2} sx={{ padding: theme.spacing(0, 2) }}>
<Grid2 xs={6}>
<Card>
<CardContent
sx={{
display: "flex",
flexDirection: "column",
}}
>
<Typography
variant="h6"
sx={{
margin: "1em 0",
fontWeight: theme.typography.fontWeightBold,
}}
>
Study Artifacts
</Typography>
<StudyArtifactCards study={studyDetail} />
</CardContent>
</Card>
</Grid2>
</Grid2>
)}
</Box>
)
}
@@ -21,7 +21,7 @@ import DeleteIcon from "@mui/icons-material/Delete"
import FullscreenIcon from "@mui/icons-material/Fullscreen"
import { actionCreator } from "../action"
import { useDeleteArtifactDialog } from "./DeleteArtifactDialog"
import { useDeleteTrialArtifactDialog } from "./DeleteArtifactDialog"
import {
useThreejsArtifactModal,
isThreejsArtifact,
@@ -31,7 +31,7 @@ import { ArtifactCardMedia } from "./ArtifactCardMedia"
export const TrialArtifactCards: FC<{ trial: Trial }> = ({ trial }) => {
const theme = useTheme()
const [openDeleteArtifactDialog, renderDeleteArtifactDialog] =
useDeleteArtifactDialog()
useDeleteTrialArtifactDialog()
const [openThreejsArtifactModal, renderThreejsArtifactModal] =
useThreejsArtifactModal()
@@ -158,7 +158,7 @@ const TrialArtifactUploader: FC<{
if (files === null) {
return
}
action.uploadArtifact(trial.study_id, trial.trial_id, files[0])
action.uploadTrialArtifact(trial.study_id, trial.trial_id, files[0])
}
const handleDrop: DragEventHandler = (e) => {
e.stopPropagation()
@@ -166,7 +166,7 @@ const TrialArtifactUploader: FC<{
const files = e.dataTransfer.files
setDragOver(false)
for (let i = 0; i < files.length; i++) {
action.uploadArtifact(trial.study_id, trial.trial_id, files[i])
action.uploadTrialArtifact(trial.study_id, trial.trial_id, files[i])
}
}
const handleDragOver: DragEventHandler = (e) => {
+1 -1
View File
@@ -12,7 +12,7 @@ def test_get_artifact_path() -> None:
def test_artifact_prefix() -> None:
actual = _backend._dashboard_trial_artifact_prefix(trial_id=0)
actual = _backend._dashboard_artifact_prefix(trial_id=0)
assert actual == "dashboard:artifacts:0:"