WeSign/unity-application
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 6 Wiki Activity

New basic signs model

Browse Source
This commit is contained in:
Jerome Coudron
2023-05-07 21:00:52 +00:00
committed by Jelle De Geest
parent 06aa9206ac
commit 43887af670
111 changed files with 952 additions and 329 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
Assets/MediaPipeUnity/Common/Scripts/KeypointManagerEmbedding.cs
View File

@@ -16,12 +16,7 @@ public class KeypointManagerEmbedding
private List<int> hand_indices = new List<int> { 0, 8, 7, 6, 5, 12, 11, 10, 9, 16, 15, 14, 13, 20, 19, 18, 17, 4, 3, 2, 1 };
private static int BUFFER_SIZE = 10;
private List<List<List<float>>> keypointsBuffer;
public KeypointManagerEmbedding()
{
keypointsBuffer = new List<List<List<float>>>();
}
private List<List<List<float>>> keypointsBuffer = new List<List<List<float>>>();
private (List<float>, List<float>) NormalizeHand(List<float> handX, List<float> handY)
{
@@ -162,6 +157,7 @@ public class KeypointManagerEmbedding
if (poseLandmarks == null || (leftHandLandmarks == null && rightHandLandmarks == null))
{
keypointsBuffer.Clear();
return;
}
@@ -259,5 +255,4 @@ public class KeypointManagerEmbedding
}
return keypointsBuffer;
}
}

13
Assets/MediaPipeUnity/ScriptableObjects/ModelList.asset
View File

@@ -16,10 +16,13 @@ MonoBehaviour:
- index: 0
modelWINDOWS: {fileID: 0}
modelMAC: {fileID: 0}
embeddingsFile: {fileID: 0}
- index: 1
modelWINDOWS: {fileID: 8538825877217656561, guid: fdbf401e965a6bf4a87637cd519f2715, type: 3}
modelMAC: {fileID: 8538825877217656561, guid: be31548ec7e7544fe9828b14657bb40b, type: 3}
modelWINDOWS: {fileID: 8538825877217656561, guid: 60511994c42f8424788e097838d5977e, type: 3}
modelMAC: {fileID: 8538825877217656561, guid: 4d4ccf4e233554d908323c3366645f94, type: 3}
embeddingsFile: {fileID: 4900000, guid: 9187978ae1cea409f994913c17e3da6d, type: 3}
- index: 2
modelWINDOWS: {fileID: 8538825877217656561, guid: fa63c40c78ba548468cad97b15cdc6c9, type: 3}
modelMAC: {fileID: 8538825877217656561, guid: 17fb70e1c284e44da8083b36bb6afcb8, type: 3}
currentModelIndex: 2
modelWINDOWS: {fileID: 8538825877217656561, guid: 1be770d7aacc847988cabc9de250aaf7, type: 3}
modelMAC: {fileID: 8538825877217656561, guid: ac007754b9aee4f59b58b4205bb75562, type: 3}
embeddingsFile: {fileID: 4900000, guid: 4e303164823194bc4be87f4c9550cfd0, type: 3}
currentModelIndex: 1

67
Assets/MediaPipeUnity/Scripts/SignPredictor.cs
View File

@@ -369,35 +369,20 @@ public class SignPredictor : MonoBehaviour
yield return new WaitUntil(() => t.IsCompleted);
model = t.Result;
modelID = modelList.GetCurrentModelIndex();
predictor_embed = new NatMLSignPredictorEmbed(model);
asyncPredictor = predictor_embed.ToAsync();
// Creating a KeypointManager
keypointManagerEmbedding = new KeypointManagerEmbedding();
if (modelID == ModelIndex.FINGERSPELLING)
{
predictor = new NatMLSignPredictor(model);
asyncPredictor = predictor.ToAsync();
// Creating a KeypointManager
keypointManager = new KeypointManager(modelInfoFile);
StartCoroutine(SignRecognitionCoroutine());
StartCoroutine(MediapipeCoroutine());
}
else
{
predictor_embed = new NatMLSignPredictorEmbed(model);
asyncPredictor = predictor_embed.ToAsync();
// Creating a KeypointManager
keypointManagerEmbedding = new KeypointManagerEmbedding();
// read the embedding data
embeddingDataList = JsonUtility.FromJson<EmbeddingDataList>($"{{\"dataList\":{modelInfoFileEmbedding}}}");
// Start the Coroutine
StartCoroutine(SignRecognitionCoroutineEmbed());
StartCoroutine(MediapipeCoroutineEmbed());
}
// read the embedding data
embeddingDataList = JsonUtility.FromJson<EmbeddingDataList>($"{{\"dataList\":{modelList.GetEmbeddings()}}}");
// Start the Coroutine
StartCoroutine(SignRecognitionCoroutineEmbed());
StartCoroutine(MediapipeCoroutineEmbed());
}
/*
/// <summary>
/// Coroutine which executes the mediapipe pipeline
/// </summary>
@@ -423,6 +408,7 @@ public class SignPredictor : MonoBehaviour
keypointManager.AddLandmarks(_poseLandmarks, _leftHandLandmarks, _rightHandLandmarks);
}
}
*/
/// <summary>
/// Coroutine which executes the mediapipe pipeline
@@ -491,6 +477,7 @@ public class SignPredictor : MonoBehaviour
return distances;
}
/*
/// <summary>
/// Coroutine which calls the sign predictor model
/// </summary>
@@ -541,6 +528,7 @@ public class SignPredictor : MonoBehaviour
}
}
*/
/// <summary>
/// Coroutine which calls the sign predictor embedding model
@@ -551,6 +539,7 @@ public class SignPredictor : MonoBehaviour
while (true)
{
List<List<List<float>>> inputData = keypointManagerEmbedding.GetKeypoints();
if (inputData != null && asyncPredictor.readyForPrediction)
{
// Getting the size of the input data
@@ -574,48 +563,32 @@ public class SignPredictor : MonoBehaviour
yield return new WaitUntil(() => task.IsCompleted);
List<float> result = task.Result;
if (0 < result.Count)
if (result.Count > 0)
{
List<DistanceEmbedding> distances = GetDistances(result, 2);
var probs = new Dictionary<string, float>();
learnableProbabilities = new Dictionary<string, float>();
for (int j = 0; j < distances.Count; j++)
{
DistanceEmbedding distanceEmbedding = distances[j];
// check if already in dictionary
if (probs.ContainsKey(distanceEmbedding.embeddingData.label_name))
if (learnableProbabilities.ContainsKey(distanceEmbedding.embeddingData.label_name))
{
// if so, check if the distance is smaller
if (probs[distanceEmbedding.embeddingData.label_name] > distanceEmbedding.distance)
if (learnableProbabilities[distanceEmbedding.embeddingData.label_name] > distanceEmbedding.distance)
{
// if so, replace the distance
probs[distanceEmbedding.embeddingData.label_name] = distanceEmbedding.distance;
learnableProbabilities[distanceEmbedding.embeddingData.label_name] = distanceEmbedding.distance;
}
}
else
{
// if not, add the distance to the dictionary
probs.Add(distanceEmbedding.embeddingData.label_name, distanceEmbedding.distance);
learnableProbabilities.Add(distanceEmbedding.embeddingData.label_name, distanceEmbedding.distance);
}
}
// convert distances to probabilities, the closer to 1.5 the better the prediction
var newProbs = new Dictionary<string, float>();
float sum = 0.0f;
foreach (KeyValuePair<string, float> entry in probs)
{
float probability = 1 / (1 + Mathf.Exp(2 * (entry.Value - 1.85f)));
newProbs.Add(entry.Key, probability);
sum += probability;
}
learnableProbabilities = new Dictionary<string, float>();
foreach (var kv in newProbs)
learnableProbabilities.Add(kv.Key, kv.Value / sum);
//UnityEngine.Debug.Log($"{learnableProbabilities.Aggregate("", (t, e) => $"{t}{e.Key}={e.Value}, ")}");
foreach (Listener listener in listeners)
{
yield return listener.ProcessIncomingCall();