WeSign/Signlanguage_Datacollector
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Making public version

Browse Source
This commit is contained in:
Victor Mylle
2023-03-09 10:15:19 +00:00
parent c0adc8bd08
commit 0e0b4794f6
11 changed files with 262 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
backend/requirements.txt
View File

@@ -9,3 +9,4 @@ requests
python-multipart
Pillow
joblib
celery

29
backend/src/models/sign.py
View File

@@ -1,7 +1,9 @@
from typing import List
import numpy as np
import requests
from pydantic import BaseModel
from sqlalchemy.ext.asyncio import AsyncSession
from sqlmodel import Field, Relationship, SQLModel
from src.exceptions.base_exception import BaseException
@@ -51,6 +53,26 @@ class Sign(SQLModelExtended, table=True):
if self.video_url is None:
raise BaseException(404, "Video url not found")
@classmethod
async def get_random(self, session: AsyncSession):
signs = await self.get_all(select_in_load=[Sign.sign_videos],session=session)
sign_videos = [len(s.sign_videos) for s in signs]
# get probability based on number of videos, lower must be more likely
# the sum must be 1
random_prob = [1 / (x + 1) for x in sign_videos]
random_prob = random_prob / np.sum(random_prob)
# get random sign
sign = np.random.choice(signs, p=random_prob)
return sign
class SignOut(BaseModel):
id: int
url: str
@@ -59,3 +81,10 @@ class SignOut(BaseModel):
video_url: str
sign_videos: List[SignVideo] = []
class SignOutSimple(BaseModel):
id: int
url: str
name: str
sign_id: str
video_url: str

9
backend/src/routers/signs.py
View File

@@ -13,7 +13,7 @@ import src.settings as settings
from src.database.database import get_session
from src.exceptions.login_exception import LoginException
from src.models.auth import Login, User
from src.models.sign import Sign, SignOut
from src.models.sign import Sign, SignOut, SignOutSimple
from src.models.signvideo import SignVideo
from src.models.token import TokenExtended
@@ -22,6 +22,12 @@ router = APIRouter(prefix="/signs")
class SignUrl(BaseModel):
url: str
@router.get("/random", status_code=status.HTTP_200_OK, response_model=SignOutSimple)
async def get_random_sign(session: AsyncSession = Depends(get_session)):
# get a random sign where there is not much data from yet
sign = await Sign.get_random(session=session)
return sign
@router.post("/", status_code=status.HTTP_201_CREATED, response_model=SignOut)
async def add_sign(url: SignUrl, Authorize: AuthJWT = Depends(), session: AsyncSession = Depends(get_session)):
Authorize.jwt_required()
@@ -127,6 +133,7 @@ async def download_all(background_tasks: BackgroundTasks, Authorize: AuthJWT = D
return response
# define a function to delete the zip file
async def delete_zip_file(zip_path):
os.remove(zip_path)

34
backend/src/routers/signvideo.py
View File

@@ -7,9 +7,11 @@ import random
import string
import subprocess
from celery import Celery
from fastapi import (APIRouter, BackgroundTasks, Depends, File, UploadFile,
status)
from fastapi.responses import FileResponse
from fastapi.concurrency import run_in_threadpool
from fastapi.responses import FileResponse, JSONResponse
from fastapi_jwt_auth import AuthJWT
from joblib import Memory
from pydantic import BaseModel
@@ -26,6 +28,9 @@ from src.utils.cryptography import verify_password
# Create a Memory object that caches data to the specified directory
cache_dir = settings.CACHE_PATH
celery_app = Celery('tasks', broker=settings.CELERY_REDIS)
async def extract_thumbnail(video_path):
proc = await asyncio.create_subprocess_exec(
"ffmpeg", "-i", video_path, "-ss", "00:00:02.000", "-vframes", "1", "-f", "image2pipe", "-"
@@ -36,15 +41,18 @@ async def extract_thumbnail(video_path):
router = APIRouter(prefix="/signs/{sign_id}/video")
@celery_app.task
def convert_video(video_filename):
command = ['ffmpeg', '-y', '-i', settings.DATA_PATH + "/" + video_filename + ".test", '-c:v', 'libx264', '-c:a', 'aac', '-strict', '-2', '-b:a', '192k', '-c:a', 'aac', '-strict', '-2', '-b:a', '192k', settings.DATA_PATH + "/" + video_filename]
# run the command
subprocess.run(command)
# delete the temporary file
os.remove(settings.DATA_PATH + "/" + video_filename + ".test")
# create the thumbnail
extract_thumbnail(settings.DATA_PATH + "/" + video_filename)
# create thumbnail
asyncio.run(extract_thumbnail(settings.DATA_PATH + "/" + video_filename))
# endpoint to upload a file and save it in the data folder
@@ -52,15 +60,14 @@ def convert_video(video_filename):
async def sign_video(
sign_id: int,
video: UploadFile,
background_tasks: BackgroundTasks,
session: AsyncSession = Depends(get_session),
Authorize: AuthJWT = Depends(),
# Authorize: AuthJWT = Depends(),
):
Authorize.jwt_required()
current_user_id: int = Authorize.get_jwt_subject()
user = await User.get_by_id(current_user_id, session)
if not user:
raise LoginException("User not found")
# Authorize.jwt_required()
# current_user_id: int = Authorize.get_jwt_subject()
# user = await User.get_by_id(current_user_id, session)
# if not user:
# raise LoginException("User not found")
sign = await Sign.get_by_id(sign_id, session)
if not sign:
raise LoginException("Sign not found")
@@ -79,12 +86,13 @@ async def sign_video(
with open(settings.DATA_PATH + "/" + video.filename + ".test", 'wb') as f:
f.write(video.file.read())
# convert the video in the background
background_tasks.add_task(convert_video, video.filename)
convert_video.delay(video.filename)
await sign_video.save(session)
return sign_video
response = JSONResponse(content={"sign_video": sign_video.dict()})
response.headers["Connection"] = "close"
return response
@router.get("/{video_id}/thumbnail/", status_code=status.HTTP_200_OK)
async def sign_video_thumbnail(

2
backend/src/settings.py
View File

@@ -17,6 +17,8 @@ DB_PORT: str = os.getenv("DB_PORT", "5432")
DB_USE_SQLITE: bool = os.getenv("DB_USE_SQLITE", False)
DB_SQLITE_PATH: str = os.getenv("DB_SQLITE_PATH", "./sqlite.db")
CELERY_REDIS: str = os.getenv("CELERY_REDIS", "redis://localhost:6379/0")
"""Storage"""
DATA_PATH: str = os.getenv("DATA_PATH", "data")
CACHE_PATH: str = os.getenv("CACHE_PATH", "cache")

8
frontend/src/App.tsx
View File

@@ -2,6 +2,7 @@ import React from 'react';
import { BrowserRouter as Router, Route, Routes } from 'react-router-dom';
import Login from './components/LoginPage';
import ProtectedRoute from './components/ProtectedRoute';
import RandomSignUpload from './components/RandomVideoUpload';
import SignDetailpage from './components/SignDetailPage';
import SignsPage from './components/SignsPage';
@@ -9,9 +10,10 @@ function App() {
return (
<Router>
<Routes>
<Route path="/" element={<ProtectedRoute><SignsPage /></ProtectedRoute>} />
<Route path="/login" element={<Login />} />
<Route path="/signs/:id" element={<ProtectedRoute><SignDetailpage /></ProtectedRoute>} />
<Route path="/" element={<RandomSignUpload />} />
<Route path="/admin/" element={<ProtectedRoute><SignsPage /></ProtectedRoute>} />
<Route path="/admin/login" element={<Login />} />
<Route path="/admin/signs/:id" element={<ProtectedRoute><SignDetailpage /></ProtectedRoute>} />
</Routes>
</Router>
);

4
frontend/src/components/ProtectedRoute.tsx
View File

@@ -10,14 +10,14 @@ const ProtectedRoute: React.FC<{ children: JSX.Element }> = ({ children }) => {
if (expired_at < new Date()) {
localStorage.removeItem('accessToken');
localStorage.removeItem('expirationDate');
return <Navigate to="/login" />;
return <Navigate to="/admin/login" />;
}
}
if (!isLoggedIn) {
return <Navigate to="/login" />;
return <Navigate to="/admin/login" />;
}
return (

175
frontend/src/components/RandomVideoUpload.tsx Executable file
View File

@@ -0,0 +1,175 @@
import React, { useState, useRef, useEffect, ChangeEvent } from 'react';
import { Sign, SignVideo, SimpleSign } from '../types/sign';
import { useParams } from 'react-router-dom';
import { getRandomSign, getSign } from '../services/signs';
import ReactModal from 'react-modal';
import { acceptVideo, deleteVideo, uploadSignVideo } from '../services/signvideos';
import { LoadingButton } from './loading-button/loading-button';
import VideoRecorder from 'react-video-recorder';
import SignVideoGrid from './SignVideoGrid';
import SignVideoPlayer from './SignVideoPlayer';
const RandomSignUpload: React.FC = () => {
const [sign, setSign] = useState<SimpleSign | null>(null);
const [signVideo, setSignVideo] = useState<string | null>(null);
const [recordedBlob, setRecordedBlob] = useState<Blob | null>(null);
const signVideoRef = useRef<HTMLVideoElement>(null);
const popupVideoRef = useRef<HTMLVideoElement>(null);
const [popUpShown, setPopUpShown] = useState(false);
const [recorderKey, setRecorderKey] = useState(1);
const [uploadProgress, setUploadProgress] = useState<number | null>(null);
const [videoUrl, setVideoUrl] = useState<string | null>(null);
useEffect(() => {
if (recordedBlob) {
setVideoUrl(URL.createObjectURL(recordedBlob));
setPopUpShown(true);
} else {
setVideoUrl(null);
}
}, [recordedBlob]);
const handleUploadProgress = (progess: number) => {
if (progess) {
setUploadProgress(progess);
}
}
const handleUpload = async () => {
setUploadProgress(0);
console.log("Uploading video...")
uploadSignVideo(sign!.id, recordedBlob!, handleUploadProgress).then((response) => {
console.log("upload complete");
setPopUpShown(false);
// get new random sign
get_random_sign();
// add the new sign video to the sign
const newSign = { ...sign! };
setSign(newSign);
setUploadProgress(100);
}).catch((error) => {
setUploadProgress(null);
}
);
}
useEffect(() => {
if (signVideoRef.current) {
signVideoRef.current.play();
}
}, []);
const get_random_sign = () => {
getRandomSign().then((response) => {
// check if the sign is different from the current one
if (sign === null || sign.id !== response.id) {
setSign(response);
// set the video url
setSignVideo(response.video_url);
} else {
// get a new sign
get_random_sign();
}
}
);
}
useEffect(() => {
// get random sign
get_random_sign();
}, []);
const dismissPopup = () => {
setPopUpShown(false);
// remove the recorded blob
setRecordedBlob(null);
};
return (
<div>
{
sign ? <div>
<div className="flex items-center justify-between">
<h1 className="text-8xl font-bold mx-auto text-center">{sign.name}</h1>
</div>
<div className="flex">
<button className="ml-auto p-2 rounded-full text-red-600 hover:bg-red-600 hover:text-white" onClick={get_random_sign}>
Skip
</button>
</div>
<div className="flex">
{
signVideo &&
<div className="w-1/2">
<video key={signVideo} loop controls width='100%' height='100%'>
<source src={signVideo} type='video/mp4' />
</video>
</div>
}
<div id="recorder-wrapper" className="w-1/2">
<VideoRecorder
key={recorderKey}
countdownTime={3000}
onRecordingComplete={(blob) => {
setRecordedBlob(blob)
// reset the VideoRecorder
setRecorderKey(prevKey => prevKey + 1);
}}
showReplayControls={false}
replayVideoAutoplayAndLoopOff={true}
isOnInitially
timeLimit={4000}
/>
</div>
</div>
</div > : <div>Loading...</div>
}
<ReactModal
isOpen={popUpShown}
shouldCloseOnOverlayClick={false}
className="modal bg-white rounded-3xl bg-gray-300 p-7"
ariaHideApp={false}
style={{
content: {
position: "absolute",
top: '50%',
left: '50%',
transform: 'translate(-50%, -50%)',
},
}}
>
{videoUrl &&
<div>
<video key="vid" ref={popupVideoRef} src={videoUrl!} controls loop className="pb-4" />
<LoadingButton title="Upload" onClick={handleUpload} progress={uploadProgress} />
</div>
}
{(uploadProgress === null || uploadProgress === 0) &&
<button onClick={dismissPopup} className="bg-white p-2 rounded-full text-red-600 hover:bg-red-600 hover:text-white absolute top-1 right-1">
<svg className="w-6 h-6" fill="none" viewBox="0 0 24 24" stroke="currentColor">
<path strokeLinecap="round" strokeLinejoin="round" strokeWidth="2" d="M6 18L18 6M6 6l12 12" />
</svg>
</button>
}
</ReactModal>
</div>
);
};
export default RandomSignUpload;

6
frontend/src/services/signs.ts
View File

@@ -88,4 +88,8 @@ const deleteSign = async (id: number) => {
return response.json();
};
export { addSign, getSigns, getSign, downloadSigns, deleteSign };
const getRandomSign = async () => {
const response = await fetch(`${process.env.REACT_APP_API_URL}/signs/random`);
return response.json();
}
export { addSign, getSigns, getSign, downloadSigns, deleteSign, getRandomSign };

6
frontend/src/services/signvideos.ts
View File

@@ -1,17 +1,17 @@
import axios from 'axios';
const uploadSignVideo = async (id: number, recordedBlob: Blob, onUploadProgress: ((arg0: number) => void)) => {
// get access token from local storage
const token = localStorage.getItem('accessToken');
let formData = new FormData();
formData.append('video', recordedBlob);
// make request to get signs
console.log("Making request")
const response = await axios.post(`${process.env.REACT_APP_API_URL}/signs/${id}/video/`, formData, {
headers: {
Authorization: `Bearer ${token}`,
ContentType: 'multipart/form-data',
// close the connection after the request is sent
Connection: 'close',
},
onUploadProgress: (progressEvent) => {
onUploadProgress(

8
frontend/src/types/sign.ts
View File

@@ -11,3 +11,11 @@ export interface Sign {
video_url: string;
sign_videos: [SignVideo];
}
export interface SimpleSign {
id: number;
url: string;
name: string;
sign_id: string;
video_url: string;
}