WeSign/unity-application
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 6 Wiki Activity

Resolve WES-97 "Integrate signpredictor in spellingbee"

Browse Source
This commit is contained in:
Dries Van Schuylenbergh
2023-03-19 17:37:50 +00:00
committed by Lukas Van Rossem
parent f827c29d3a
commit 3abc24a39c
72 changed files with 3169 additions and 1886 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
Assets/MediaPipeUnity/Common/Scripts/KeypointManager.cs
View File

@@ -1,7 +1,7 @@
using System.Collections;
using Mediapipe;
using System.Collections.Generic;
using UnityEngine;
using System.Linq;
using UnityEngine;
public class ModelInfo
{
@@ -12,17 +12,16 @@ public class ModelInfo
public class KeypointManager
{
private ModelInfo model_info;
private List<List<float>> keypoints_buffer;
private ModelInfo modelInfo;
private List<List<float>> keypointsBuffer;
public KeypointManager()
public KeypointManager(TextAsset modelInfoFile)
{
TextAsset model_info_json = Resources.Load<TextAsset>("Models/FingerSpelling/landmarks");
this.model_info = JsonUtility.FromJson<ModelInfo>(model_info_json.text);
this.keypoints_buffer = new List<List<float>>();
modelInfo = JsonUtility.FromJson<ModelInfo>(modelInfoFile.text);
keypointsBuffer = new List<List<float>>();
}
private (List<float>, List<float>) normalizeHand(List<float> hand_x, List<float> hand_y)
private (List<float>, List<float>) NormalizeHand(List<float> hand_x, List<float> hand_y)
{
float min_x = hand_x.Min();
@@ -54,7 +53,7 @@ public class KeypointManager
return (normalized_x, normalized_y);
}
public void addLandmarks(Mediapipe.NormalizedLandmarkList poseLandmarks, Mediapipe.NormalizedLandmarkList leftHandLandmarks, Mediapipe.NormalizedLandmarkList rightHandLandmarks)
public void AddLandmarks(NormalizedLandmarkList poseLandmarks, NormalizedLandmarkList leftHandLandmarks, NormalizedLandmarkList rightHandLandmarks)
{
List<float> pose_x = new List<float>();
List<float> pose_y = new List<float>();
@@ -65,7 +64,7 @@ public class KeypointManager
if (poseLandmarks != null)
{
foreach (var landmark_index in model_info.pose_landmarks)
foreach (int landmark_index in modelInfo.pose_landmarks)
{
pose_x.Add(poseLandmarks.Landmark[landmark_index].X);
pose_y.Add(poseLandmarks.Landmark[landmark_index].Y);
@@ -73,7 +72,7 @@ public class KeypointManager
}
else
{
foreach (var landmark_index in model_info.pose_landmarks)
foreach (int _ in modelInfo.pose_landmarks)
{
pose_x.Add(0);
pose_y.Add(0);
@@ -82,7 +81,7 @@ public class KeypointManager
foreach (var landmark_index in model_info.hand_landmarks)
foreach (int landmark_index in modelInfo.hand_landmarks)
{
if (leftHandLandmarks == null)
{
@@ -107,13 +106,15 @@ public class KeypointManager
}
// TODO: Add normalization
(left_hand_x, left_hand_y) = normalizeHand(left_hand_x, left_hand_y);
(right_hand_x, right_hand_y) = normalizeHand(right_hand_x, right_hand_y);
//Debug.Log($"pose_landMarks = [{modelInfo.pose_landmarks.Aggregate(" ", (t, f) => $"{t} {f}")}]");
//Debug.Log($"hand_landmarks = [{modelInfo.hand_landmarks.Aggregate(" ", (t, f) => $"{t} {f}")}]");
(left_hand_x, left_hand_y) = NormalizeHand(left_hand_x, left_hand_y);
(right_hand_x, right_hand_y) = NormalizeHand(right_hand_x, right_hand_y);
if (keypoints_buffer.Count >= 10)
if (keypointsBuffer.Count >= 10)
{
keypoints_buffer.RemoveAt(0);
keypointsBuffer.RemoveAt(0);
}
List<float> keypoints = new List<float>();
@@ -133,15 +134,15 @@ public class KeypointManager
keypoints.Add(right_hand_y[i]);
}
keypoints_buffer.Add(keypoints);
keypointsBuffer.Add(keypoints);
}
public List<List<float>> getAllKeypoints()
public List<List<float>> GetKeypoints()
{
if (keypoints_buffer.Count < 10)
if (keypointsBuffer.Count < 10)
{
return null;
}
return keypoints_buffer;
return keypointsBuffer;
}
}