Resolve WES-97 "Integrate signpredictor in spellingbee"

Assets/MediaPipeUnity/Scripts/Feedback.cs

@@ -0,0 +1,180 @@
+using DigitalRuby.Tween;
+using Mediapipe.Unity.Tutorial;
+using System;
+using System.Collections;
+using TMPro;
+using UnityEngine;
+using UnityEngine.Events;
+using UnityEngine.UI;
+
+/// <summary>
+/// Class to display feedback during a course
+/// </summary>
+public class Feedback : MonoBehaviour
+{
+    /// <summary>
+    /// Reference to the feedback field
+    /// </summary>
+    public TMP_Text feedbackText;
+
+    /// <summary>
+    /// Reference to the progress bar
+    /// </summary>
+    public Slider feedbackProgress;
+
+    /// <summary>
+    /// Reference to the progress bar image, so we can add fancy colors
+    /// </summary>
+    public Image feedbackProgressImage;
+
+    /// <summary>
+    /// Reference to the sign predictor
+    /// </summary>
+    public SignPredictor signPredictor;
+
+    /// <summary>
+    /// Callback for getting the correct sign
+    /// </summary>
+    public Func<string> getSignCallback;
+
+    /// <summary>
+    /// Callback to initiate the next sign
+    /// </summary>
+    public UnityAction<string> predictSignCallback;
+
+    /// <summary>
+    /// Timer to keep track of how long a incorrect sign is performed
+    /// </summary>
+    private DateTime timer;
+
+    /// <summary>
+    /// Current predicted sign
+    /// </summary>
+    private string predictedSign = null;
+
+    /// <summary>
+    /// Previous incorrect sign, so we can keep track whether the user is wrong or the user is still changing signs
+    /// </summary>
+    private string previousIncorrectSign = null;
+
+    /// <summary>
+    /// Start is called before the first frame update
+    /// </summary>
+    void Start()
+    {
+        // Start the coroutine to update the scale every 200 milliseconds
+        StartCoroutine(UpdateFeedback());
+    }
+
+    /// <summary>
+    /// UpdateScale updates the progress bar every 200ms, updated the feedback text, and progress bar color
+    /// If a high enough accuracy is detected, it will go to the next sign
+    /// </summary>
+    /// <returns></returns>
+    IEnumerator UpdateFeedback()
+    {
+        while (true)
+        {
+            if (getSignCallback != null && predictSignCallback != null)
+            {
+
+                // Get current sign
+                string currentSign = getSignCallback();
+
+                // Get the predicted sign
+                if (signPredictor != null && signPredictor.learnableProbabilities != null &&
+                    currentSign != null && signPredictor.learnableProbabilities.ContainsKey(currentSign))
+                {
+                    float accuracy = signPredictor.learnableProbabilities[currentSign];
+                    if (accuracy > 0.98)
+                    {
+                        // TODO: fix emojis
+                        feedbackText.text = "✨ Perfect ✨";
+                        Color col = new Color(0xff / 255.0f, 0xcc / 255.0f, 0x00 / 255.0f);
+                        feedbackText.color = col;
+                        feedbackProgressImage.color = col;
+                    }
+                    else if (accuracy > 0.95)
+                    {
+                        feedbackText.text = "Super!";
+                        Color col = new Color(0x00 / 255.0f, 0xff / 255.0f, 0xcc / 255.0f);
+                        feedbackText.color = col;
+                        feedbackProgressImage.color = col;
+                    }
+                    else if (accuracy > 0.90)
+                    {
+                        feedbackText.text = "Goed";
+                        feedbackText.color = Color.green;
+                        feedbackProgressImage.color = Color.green;
+                    }
+                    else if (accuracy > 0.80)
+                    {
+                        feedbackText.text = "Bijna...";
+                        Color col = new Color(0xff / 255.0f, 0x66 / 255.0f, 0x00 / 255.0f);
+                        feedbackText.color = col;
+                        feedbackProgressImage.color = col;
+                    }
+                    else
+                    {
+                        feedbackText.text = "Detecteren...";
+                        feedbackText.color = Color.red;
+                        feedbackProgressImage.color = Color.red;
+                    }
+
+                    float oldValue = feedbackProgress.value;
+                    // use an exponential scale 
+                    float newValue = Mathf.Exp(4 * (accuracy - 1.0f));
+                    feedbackProgress.gameObject.Tween("FeedbackUpdate", oldValue, newValue, 0.2f, TweenScaleFunctions.CubicEaseInOut, (t) =>
+                    {
+                        if (feedbackProgress != null)
+                        {
+                            feedbackProgress.value = t.CurrentValue;
+                        }
+                    });
+
+                    // Check whether (in)correct sign has high accuracy
+                    foreach (var kv in signPredictor.learnableProbabilities)
+                    {
+                        if (kv.Value > 0.90)
+                        {
+                            predictedSign = kv.Key;
+                            // Correct sign
+                            if (predictedSign == currentSign)
+                            {
+                                yield return new WaitForSeconds(1.0f);
+                                predictSignCallback(predictedSign);
+                                timer = DateTime.Now;
+                                predictedSign = null;
+                                previousIncorrectSign = null;
+                            }
+                            // Incorrect sign
+                            else
+                            {
+                                if (previousIncorrectSign != predictedSign)
+                                {
+                                    timer = DateTime.Now;
+                                    previousIncorrectSign = predictedSign;
+                                }
+                                else if (DateTime.Now - timer > TimeSpan.FromSeconds(2.0f))
+                                {
+                                    predictSignCallback(predictedSign);
+                                    timer = DateTime.Now;
+                                    predictedSign = null;
+                                    previousIncorrectSign = null;
+                                }
+                            }
+                            break;
+                        }
+                    }
+                }
+                else
+                {
+                    feedbackProgress.value = 0.0f;
+                }
+            }
+
+            // Wait for 200 milliseconds before updating the scale again
+            yield return new WaitForSeconds(0.2f);
+        }
+    }
+}