WeSign/unity-application
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 6 Wiki Activity

Sprint 6

Browse Source
This commit is contained in:
Jelle De Geest
2023-05-24 21:08:16 +02:00
parent 2a172bdb28
commit 163eb40c0f
1052 changed files with 20520 additions and 17253 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
Assets/MediaPipeUnity/Scripts/SignPredictor.asmdef
View File

@@ -8,7 +8,8 @@
"GUID:58e104b97fb3752438ada2902a36dcbf",
"GUID:7f2d0ee6dd21e1d4eb25b71b7a749d25",
"GUID:f55a02e98b01bc849b30d9650ccd8f15",
"GUID:d23f64cfd3b314bb4a18a8284c99bf5e"
"GUID:d23f64cfd3b314bb4a18a8284c99bf5e",
"GUID:e83ddf9a537a96b4a804a16bb7872ec1"
],
"includePlatforms": [],
"excludePlatforms": [],

176
Assets/MediaPipeUnity/Scripts/SignPredictor.cs
View File

@@ -10,7 +10,9 @@ using System.Threading.Tasks;
using UnityEngine;
using UnityEngine.UI;
/// <summary>
/// Class for EmbeddingData, Embeddings are used in the Model to make prediction
/// </summary>
[System.Serializable]
public class EmbeddingData
{
@@ -19,26 +21,45 @@ public class EmbeddingData
public int labels;
}
/// <summary>
/// Class for a list of EmbeddingData
/// </summary>
[System.Serializable]
public class EmbeddingDataList
{
public List<EmbeddingData> dataList;
}
/// <summary>
/// Class to save the distance of an embedding
/// </summary>
public class DistanceEmbedding
{
public float distance;
public EmbeddingData embeddingData;
/// <summary>
/// Creation of DistanceEmbedding
/// </summary>
/// <param name="distance"></param>
/// <param name="embeddingData"></param>
public DistanceEmbedding(float distance, EmbeddingData embeddingData)
{
this.distance = distance;
this.embeddingData = embeddingData;
}
}
/// <summary>
/// Class to compare the distance of two embeddings
/// </summary>
public class DistanceComparer : IComparer<DistanceEmbedding>
{
/// <summary>
/// Function to compare the distance of two DistanceEmbeddings
/// </summary>
/// <param name="x"></param>
/// <param name="y"></param>
/// <returns></returns>
public int Compare(DistanceEmbedding x, DistanceEmbedding y)
{
return x.distance.CompareTo(y.distance);
@@ -46,7 +67,7 @@ public class DistanceComparer : IComparer<DistanceEmbedding>
}
/// <summary>
///
/// Sign predictor class which gives input to the games/courses by extractin information from the webcam
/// </summary>
public class SignPredictor : MonoBehaviour
{
@@ -95,7 +116,6 @@ public class SignPredictor : MonoBehaviour
return predictions;
}
/// <summary>
/// Disposing the MLEdgeModel
/// </summary>
@@ -153,7 +173,6 @@ public class SignPredictor : MonoBehaviour
return predictions;
}
/// <summary>
/// Disposing the MLEdgeModel
/// </summary>
@@ -163,6 +182,9 @@ public class SignPredictor : MonoBehaviour
}
}
/// <summary>
/// List of listeners that want to get notified on new predictions
/// </summary>
public List<Listener> listeners = new List<Listener>();
/// <summary>
@@ -170,9 +192,6 @@ public class SignPredictor : MonoBehaviour
/// </summary>
private NatMLSignPredictorEmbed predictor_embed;
private NatMLSignPredictor predictor;
/// <summary>
/// The asynchronous predictor which is used to predict the sign using an MLEdgemodel
/// </summary>
@@ -197,6 +216,10 @@ public class SignPredictor : MonoBehaviour
/// Reference to the model info file
/// </summary>
public TextAsset modelInfoFile;
/// <summary>
/// Reference to the model Embedding file
/// </summary>
public TextAsset modelInfoFileEmbedding;
/// <summary>
@@ -296,11 +319,11 @@ public class SignPredictor : MonoBehaviour
/// </summary>
private static bool resourceManagerIsInitialized = false;
private List<string> signs;
/// <summary>
/// List of the EmbeddingData
/// </summary>
private EmbeddingDataList embeddingDataList;
private ModelIndex modelID;
/// <summary>
/// Google Mediapipe setup & run
/// </summary>
@@ -364,8 +387,19 @@ public class SignPredictor : MonoBehaviour
// Check if a model is ready to load
yield return new WaitUntil(() => modelList.HasValidModel());
// Create Model
Task<MLEdgeModel> t = Task.Run(() => MLEdgeModel.Create(modelList.GetCurrentModel()));
NatML.MLEdgeModel.Configuration myConfig = null;
// Create Configuration
if (PersistentDataController.GetInstance().IsUsingGPU())
{
// Create a new instance of the Configuration class
myConfig = new NatML.MLEdgeModel.Configuration();
// Set the computeTarget property to GPU
myConfig.computeTarget = NatML.MLEdgeModel.ComputeTarget.GPU;
}
Task<MLEdgeModel> t = Task.Run(() => MLEdgeModel.Create(modelList.GetCurrentModel(), myConfig));
yield return new WaitUntil(() => t.IsCompleted);
model = t.Result;
@@ -382,34 +416,6 @@ public class SignPredictor : MonoBehaviour
StartCoroutine(MediapipeCoroutineEmbed());
}
/*
/// <summary>
/// Coroutine which executes the mediapipe pipeline
/// </summary>
/// <returns></returns>
private IEnumerator MediapipeCoroutine()
{
while (true)
{
inputTexture.SetPixels32(webcamTexture.GetPixels32(pixelData));
var imageFrame = new ImageFrame(ImageFormat.Types.Format.Srgba, width, height, width * 4, inputTexture.GetRawTextureData<byte>());
var currentTimestamp = stopwatch.ElapsedTicks / (System.TimeSpan.TicksPerMillisecond / 1000);
graph.AddPacketToInputStream("input_video", new ImageFramePacket(imageFrame, new Timestamp(currentTimestamp))).AssertOk();
yield return new WaitForEndOfFrame();
NormalizedLandmarkList _poseLandmarks = null;
NormalizedLandmarkList _leftHandLandmarks = null;
NormalizedLandmarkList _rightHandLandmarks = null;
yield return new WaitUntil(() => { posestream.TryGetNext(out _poseLandmarks); return true; });
yield return new WaitUntil(() => { leftstream.TryGetNext(out _leftHandLandmarks); return true; });
yield return new WaitUntil(() => { rightstream.TryGetNext(out _rightHandLandmarks); return true; });
keypointManager.AddLandmarks(_poseLandmarks, _leftHandLandmarks, _rightHandLandmarks);
}
}
*/
/// <summary>
/// Coroutine which executes the mediapipe pipeline
/// </summary>
@@ -436,7 +442,14 @@ public class SignPredictor : MonoBehaviour
}
}
/// <summary>
/// This function calculates the Minkowski distance between two points in a p-dimensional space.
/// The Minkowski distance is a generalization of Euclidean and Manhattan distances, depending on the value of p.
/// </summary>
/// <param name="x">List of float values representing the coordinates of the first point.</param>
/// <param name="y">Array of float values representing the coordinates of the second point.</param>
/// <param name="p">Integer value representing the power parameter of the Minkowski distance. When p=2, it calculates Euclidean distance, and when p=1, it calculates Manhattan distance.</param>
/// <returns>Returns the Minkowski distance between two points x and y in a p-dimensional space.</returns>
private float MinkowskiDistance(List<float> x, float[] y, int p)
{
int dimensions = x.Count;
@@ -450,6 +463,13 @@ public class SignPredictor : MonoBehaviour
return Mathf.Pow(sum, 1.0f / p);
}
/// <summary>
/// This function calculates the Minkowski distances between a given embedding and a list of predefined embeddings.
/// The function returns a sorted list of distance and associated embedding data, sorted in ascending order of distance.
/// </summary>
/// <param name="embedding">A list of float values representing the embedding for which distances to all embeddings in the dataList need to be computed.</param>
/// <param name="p">An optional integer parameter representing the power parameter of the Minkowski distance. Defaults to 2, implying Euclidean distance is calculated if not specified.</param>
/// <returns>Returns a list of DistanceEmbedding objects, each representing the distance between the given embedding and an embedding from the dataList, along with the associated EmbeddingData. The list is sorted in ascending order of distance.</returns>
private List<DistanceEmbedding> GetDistances(List<float> embedding, int p = 2)
{
List<DistanceEmbedding> distances = new List<DistanceEmbedding>();
@@ -477,59 +497,6 @@ public class SignPredictor : MonoBehaviour
return distances;
}
/*
/// <summary>
/// Coroutine which calls the sign predictor model
/// </summary>
/// <returns></returns>
private IEnumerator SignRecognitionCoroutine()
{
while (true)
{
List<List<float>> inputData = keypointManager.GetKeypoints();
if (inputData != null && asyncPredictor.readyForPrediction)
{
// Getting the size of the input data
int framecount = inputData.Count;
int keypointsPerFrame = inputData[0].Count;
// Creating ArrayFeature
int[] shape = { framecount, keypointsPerFrame };
float[] input = new float[framecount * keypointsPerFrame];
int i = 0;
inputData.ForEach((e) => e.ForEach((f) => input[i++] = f));
MLArrayFeature<float> feature = new MLArrayFeature<float>(input, shape);
// Predicting
Task<List<float>> task = Task.Run(async () => await asyncPredictor.Predict(feature));
yield return new WaitUntil(() => task.IsCompleted);
List<float> result = task.Result;
if (0 < result.Count)
{
learnableProbabilities = new Dictionary<string, float>();
for (int j = 0; j < result.Count; j++)
{
learnableProbabilities.Add(signs[j].ToUpper(), result[j]);
}
foreach (Listener listener in listeners)
{
yield return listener.ProcessIncomingCall();
}
}
else
{
// Wait until next frame
yield return null;
}
}
yield return null;
}
}
*/
/// <summary>
/// Coroutine which calls the sign predictor embedding model
/// </summary>
@@ -553,13 +520,10 @@ public class SignPredictor : MonoBehaviour
int i = 0;
inputData.ForEach((e) => e.ForEach((f) => f.ForEach((k) => input[i++] = k)));
MLArrayFeature<float> feature = new MLArrayFeature<float>(input, shape);
// Predicting
Task<List<float>> task = Task.Run(async () => await asyncPredictor.Predict(feature));
yield return new WaitUntil(() => task.IsCompleted);
List<float> result = task.Result;
@@ -589,17 +553,17 @@ public class SignPredictor : MonoBehaviour
}
}
//UnityEngine.Debug.Log(learnableProbabilities.Aggregate("", (t, d) => $"{t}{d}, "));
foreach (Listener listener in listeners)
{
yield return listener.ProcessIncomingCall();
}
}
}
yield return null;
}
}
/// <summary>
@@ -657,7 +621,10 @@ public class SignPredictor : MonoBehaviour
}
}
/// <summary>
/// Let the class know which Model it should load for the chosen Game/Course
/// </summary>
/// <param name="index"></param>
public void SetModel(ModelIndex index)
{
this.modelList.SetCurrentModel(index);
@@ -669,8 +636,6 @@ public class SignPredictor : MonoBehaviour
public void SwapScreen(RawImage screen)
{
this.screen = screen;
//width = webcamTexture.width;
//height = webcamTexture.height;
if (webcamTexture != null)
{
float webcamAspect = (float)webcamTexture.width / (float)webcamTexture.height;
@@ -678,9 +643,4 @@ public class SignPredictor : MonoBehaviour
this.screen.texture = webcamTexture;
}
}
public void SetSignsList(List<string> signs)
{
this.signs = signs;
}
}