WeSign/unity-application
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 6 Wiki Activity

Resolve WES-131-Feedback-REfactor

Browse Source
This commit is contained in:
Jerome Coudron
2023-04-02 12:27:59 +00:00
committed by Dries Van Schuylenbergh
parent b955d2164c
commit a808e73a29
27 changed files with 663 additions and 560 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
Assets/MediaPipeUnity/Scripts/AbstractFeedback.cs Normal file
View File

@@ -0,0 +1,40 @@
using DigitalRuby.Tween;
using Mediapipe.Unity.Tutorial;
using System;
using System.Collections;
using TMPro;
using UnityEngine;
using UnityEngine.Events;
using UnityEngine.UI;
/// <summary>
/// Class to display feedback during a course
/// </summary>
public abstract class AbstractFeedback : MonoBehaviour, Listener
{
/// <summary>
/// Reference to the sign predictor
/// </summary>
public SignPredictor signPredictor;
/// <summary>
/// The function that is called by the publisher on all its listeners
/// </summary>
/// <returns></returns>
public IEnumerator ProcessIncomingCall()
{
yield return StartCoroutine(UpdateFeedback());
}
/// <summary>
/// A function to add yourself as listener to the signPredictor you are holding
/// </summary>
public void AddSelfAsListener()
{
signPredictor.listeners.Add(this);
}
/// <summary>
/// The function that holds the logic to process the new probabilities of the signPredictor
/// </summary>
/// <returns></returns>
protected abstract IEnumerator UpdateFeedback();
}

2
Assets/MediaPipeUnity/Scripts/Feedback.cs.meta → Assets/MediaPipeUnity/Scripts/AbstractFeedback.cs.meta
View File

@@ -1,5 +1,5 @@
fileFormatVersion: 2
guid: 44e682a32ee15cc489bf50f3a06f717b
guid: 7b5ac794337a54143a6e3077483d96c9
MonoImporter:
externalObjects: {}
serializedVersion: 2

182
Assets/MediaPipeUnity/Scripts/Feedback.cs
View File

@@ -1,182 +0,0 @@
using DigitalRuby.Tween;
using Mediapipe.Unity.Tutorial;
using System;
using System.Collections;
using TMPro;
using UnityEngine;
using UnityEngine.Events;
using UnityEngine.UI;
/// <summary>
/// Class to display feedback during a course
/// </summary>
public class Feedback : MonoBehaviour
{
/// <summary>
/// Reference to the feedback field
/// </summary>
public TMP_Text feedbackText;
/// <summary>
/// Reference to the progress bar
/// </summary>
public Slider feedbackProgress;
/// <summary>
/// Reference to the progress bar image, so we can add fancy colors
/// </summary>
public Image feedbackProgressImage;
/// <summary>
/// Reference to the sign predictor
/// </summary>
public SignPredictor signPredictor;
/// <summary>
/// Callback for getting the correct sign
/// </summary>
public Func<string> getSignCallback;
/// <summary>
/// Callback to initiate the next sign
/// </summary>
public UnityAction<string> predictSignCallback;
/// <summary>
/// Timer to keep track of how long a incorrect sign is performed
/// </summary>
private DateTime timer;
/// <summary>
/// Current predicted sign
/// </summary>
private string predictedSign = null;
/// <summary>
/// Previous incorrect sign, so we can keep track whether the user is wrong or the user is still changing signs
/// </summary>
private string previousIncorrectSign = null;
/// <summary>
/// Start is called before the first frame update
/// </summary>
void Start()
{
// Start the coroutine to update the scale every 200 milliseconds
StartCoroutine(UpdateFeedback());
}
/// <summary>
/// UpdateScale updates the progress bar every 200ms, updated the feedback text, and progress bar color
/// If a high enough accuracy is detected, it will go to the next sign
/// </summary>
/// <returns></returns>
IEnumerator UpdateFeedback()
{
while (true)
{
if (getSignCallback != null && predictSignCallback != null)
{
// Get current sign
string currentSign = getSignCallback();
// Get the predicted sign
if (signPredictor != null && signPredictor.learnableProbabilities != null &&
currentSign != null && signPredictor.learnableProbabilities.ContainsKey(currentSign))
{
float accuracy = signPredictor.learnableProbabilities[currentSign];
if (feedbackText != null && feedbackProgressImage != null){
if (accuracy > 0.98)
{
// TODO: fix emojis
feedbackText.text = "✨ Perfect ✨";
Color col = new Color(0xff / 255.0f, 0xcc / 255.0f, 0x00 / 255.0f);
feedbackText.color = col;
feedbackProgressImage.color = col;
}
else if (accuracy > 0.95)
{
feedbackText.text = "Super!";
Color col = new Color(0x00 / 255.0f, 0xff / 255.0f, 0xcc / 255.0f);
feedbackText.color = col;
feedbackProgressImage.color = col;
}
else if (accuracy > 0.90)
{
feedbackText.text = "Goed";
feedbackText.color = Color.green;
feedbackProgressImage.color = Color.green;
}
else if (accuracy > 0.80)
{
feedbackText.text = "Bijna...";
Color col = new Color(0xff / 255.0f, 0x66 / 255.0f, 0x00 / 255.0f);
feedbackText.color = col;
feedbackProgressImage.color = col;
}
else
{
feedbackText.text = "Detecteren...";
feedbackText.color = Color.red;
feedbackProgressImage.color = Color.red;
}
float oldValue = feedbackProgress.value;
// use an exponential scale
float newValue = Mathf.Exp(4 * (accuracy - 1.0f));
feedbackProgress.gameObject.Tween("FeedbackUpdate", oldValue, newValue, 0.2f, TweenScaleFunctions.CubicEaseInOut, (t) =>
{
if (feedbackProgress != null)
{
feedbackProgress.value = t.CurrentValue;
}
});
}
// Check whether (in)correct sign has high accuracy
foreach (var kv in signPredictor.learnableProbabilities)
{
if (kv.Value > 0.90)
{
predictedSign = kv.Key;
// Correct sign
if (predictedSign == currentSign)
{
yield return new WaitForSeconds(1.0f);
predictSignCallback(predictedSign);
timer = DateTime.Now;
predictedSign = null;
previousIncorrectSign = null;
}
// Incorrect sign
else
{
if (previousIncorrectSign != predictedSign)
{
timer = DateTime.Now;
previousIncorrectSign = predictedSign;
}
else if (DateTime.Now - timer > TimeSpan.FromSeconds(2.0f))
{
predictSignCallback(predictedSign);
timer = DateTime.Now;
predictedSign = null;
previousIncorrectSign = null;
}
}
break;
}
}
}
else if(feedbackProgress != null)
{
feedbackProgress.value = 0.0f;
}
}
// Wait for 200 milliseconds before updating the scale again
yield return new WaitForSeconds(0.2f);
}
}
}

3
Assets/MediaPipeUnity/Scripts/SignPredictor.asmdef
View File

@@ -7,7 +7,8 @@
"GUID:04c4d86a70aa56c55a78c61f1ab1a56d",
"GUID:edc93f477bb73a743a97d6882ed330b3",
"GUID:58e104b97fb3752438ada2902a36dcbf",
"GUID:7f2d0ee6dd21e1d4eb25b71b7a749d25"
"GUID:7f2d0ee6dd21e1d4eb25b71b7a749d25",
"GUID:f55a02e98b01bc849b30d9650ccd8f15"
],
"includePlatforms": [],
"excludePlatforms": [],

103
Assets/MediaPipeUnity/Scripts/SignPredictor.cs
View File

@@ -141,6 +141,8 @@ namespace Mediapipe.Unity.Tutorial
/// </summary>
private Tensor inputTensor;
public List<Listener> listeners = new List<Listener>();
/// <summary>
/// Google Mediapipe setup & run
/// </summary>
@@ -159,6 +161,7 @@ namespace Mediapipe.Unity.Tutorial
webcamTexture.Play();
yield return new WaitUntil(() => webcamTexture.width > 16);
// Set webcam aspect ratio
@@ -167,63 +170,63 @@ namespace Mediapipe.Unity.Tutorial
float webcamAspect = (float)webcamTexture.width / (float)webcamTexture.height;
screen.rectTransform.sizeDelta = new Vector2(screen.rectTransform.sizeDelta.y * webcamAspect, (screen.rectTransform.sizeDelta.y));
screen.texture = webcamTexture;
if(screen2 != null)
if (screen2 != null)
{
screen2.rectTransform.sizeDelta = new Vector2(screen2.rectTransform.sizeDelta.y * webcamAspect, (screen2.rectTransform.sizeDelta.y));
}
// TODO this method is kinda meh you should use
inputTexture = new Texture2D(width, height, TextureFormat.RGBA32, false);
pixelData = new Color32[width * height];
if (!resourceManagerIsInitialized)
if (modelList.GetCurrentModel() != null)
{
resourceManager = new StreamingAssetsResourceManager();
yield return resourceManager.PrepareAssetAsync("pose_detection.bytes");
yield return resourceManager.PrepareAssetAsync("pose_landmark_full.bytes");
yield return resourceManager.PrepareAssetAsync("face_landmark.bytes");
yield return resourceManager.PrepareAssetAsync("hand_landmark_full.bytes");
yield return resourceManager.PrepareAssetAsync("face_detection_short_range.bytes");
yield return resourceManager.PrepareAssetAsync("hand_recrop.bytes");
yield return resourceManager.PrepareAssetAsync("handedness.txt");
resourceManagerIsInitialized = true;
// TODO this method is kinda meh you should use
inputTexture = new Texture2D(width, height, TextureFormat.RGBA32, false);
pixelData = new Color32[width * height];
if (!resourceManagerIsInitialized)
{
resourceManager = new StreamingAssetsResourceManager();
yield return resourceManager.PrepareAssetAsync("pose_detection.bytes");
yield return resourceManager.PrepareAssetAsync("pose_landmark_full.bytes");
yield return resourceManager.PrepareAssetAsync("face_landmark.bytes");
yield return resourceManager.PrepareAssetAsync("hand_landmark_full.bytes");
yield return resourceManager.PrepareAssetAsync("face_detection_short_range.bytes");
yield return resourceManager.PrepareAssetAsync("hand_recrop.bytes");
yield return resourceManager.PrepareAssetAsync("handedness.txt");
resourceManagerIsInitialized = true;
}
stopwatch = new Stopwatch();
// Setting up the graph
graph = new CalculatorGraph(configAsset.text);
posestream = new OutputStream<NormalizedLandmarkListPacket, NormalizedLandmarkList>(graph, "pose_landmarks", "pose_landmarks_presence");
leftstream = new OutputStream<NormalizedLandmarkListPacket, NormalizedLandmarkList>(graph, "left_hand_landmarks", "left_hand_landmarks_presence");
rightstream = new OutputStream<NormalizedLandmarkListPacket, NormalizedLandmarkList>(graph, "right_hand_landmarks", "right_hand_landmarks_presence");
posestream.StartPolling().AssertOk();
leftstream.StartPolling().AssertOk();
rightstream.StartPolling().AssertOk();
graph.StartRun().AssertOk();
stopwatch.Start();
keypointManager = new KeypointManager(modelInfoFile);
// check if model exists at path
//var model = ModelLoader.Load(Resources.Load<NNModel>("Models/Fingerspelling/model_A-L"));
worker = modelList.GetCurrentModel().CreateWorker();
StartCoroutine(SignRecognitionCoroutine());
StartCoroutine(MediapipeCoroutine());
}
stopwatch = new Stopwatch();
// Setting up the graph
graph = new CalculatorGraph(configAsset.text);
posestream = new OutputStream<NormalizedLandmarkListPacket, NormalizedLandmarkList>(graph, "pose_landmarks", "pose_landmarks_presence");
leftstream = new OutputStream<NormalizedLandmarkListPacket, NormalizedLandmarkList>(graph, "left_hand_landmarks", "left_hand_landmarks_presence");
rightstream = new OutputStream<NormalizedLandmarkListPacket, NormalizedLandmarkList>(graph, "right_hand_landmarks", "right_hand_landmarks_presence");
posestream.StartPolling().AssertOk();
leftstream.StartPolling().AssertOk();
rightstream.StartPolling().AssertOk();
graph.StartRun().AssertOk();
stopwatch.Start();
keypointManager = new KeypointManager(modelInfoFile);
// check if model exists at path
//var model = ModelLoader.Load(Resources.Load<NNModel>("Models/Fingerspelling/model_A-L"));
worker = modelList.GetCurrentModel().CreateWorker();
StartCoroutine(SignRecognitionCoroutine());
StartCoroutine(MediapipeCoroutine());
}
public void ChangeModel(ModelIndex index)
/// <summary>
/// Called at the start of course/Minigame, will set the model before the start of SIgnPredictor is called.
/// </summary>
/// <param name="index">The index of the model to be used</param>
public void SetModel(ModelIndex index)
{
this.modelList.SetCurrentModel(index);
// If a worker already existed, we throw it out
worker?.Dispose();
// Add a new worker for the new model
worker = modelList.GetCurrentModel().CreateWorker();
}
/// <summary>
@@ -325,6 +328,10 @@ namespace Mediapipe.Unity.Tutorial
learnableProbabilities.Add(((char)(i + 65)).ToString(), softmaxedOutput2[i]);
}
//Debug.Log($"prob = [{learnableProbabilities.Aggregate(" ", (t, kv) => $"{t}{kv.Key}:{kv.Value} ")}]");
foreach(Listener listener in listeners)
{
yield return listener.ProcessIncomingCall();
}
}
else
{