unity-application/Assets/MediaPipeUnity/Scripts/Feedback.cs

using DigitalRuby.Tween;
using Mediapipe.Unity.Tutorial;
using System;
using System.Collections;
using TMPro;
using UnityEngine;
using UnityEngine.Events;
using UnityEngine.UI;

/// <summary>
/// Class to display feedback during a course
/// </summary>
public class Feedback : MonoBehaviour
{
    /// <summary>
    /// Reference to the feedback field
    /// </summary>
    public TMP_Text feedbackText;

    /// <summary>
    /// Reference to the progress bar
    /// </summary>
    public Slider feedbackProgress;

    /// <summary>
    /// Reference to the progress bar image, so we can add fancy colors
    /// </summary>
    public Image feedbackProgressImage;

    /// <summary>
    /// Reference to the sign predictor
    /// </summary>
    public SignPredictor signPredictor;

    /// <summary>
    /// Callback for getting the correct sign
    /// </summary>
    public Func<string> getSignCallback;

    /// <summary>
    /// Callback to initiate the next sign
    /// </summary>
    public UnityAction<string> predictSignCallback;

    /// <summary>
    /// Timer to keep track of how long a incorrect sign is performed
    /// </summary>
    private DateTime timer;

    /// <summary>
    /// Current predicted sign
    /// </summary>
    private string predictedSign = null;

    /// <summary>
    /// Previous incorrect sign, so we can keep track whether the user is wrong or the user is still changing signs
    /// </summary>
    private string previousIncorrectSign = null;

    /// <summary>
    /// Start is called before the first frame update
    /// </summary>
    void Start()
    {
        // Start the coroutine to update the scale every 200 milliseconds
        StartCoroutine(UpdateFeedback());
    }

    /// <summary>
    /// UpdateScale updates the progress bar every 200ms, updated the feedback text, and progress bar color
    /// If a high enough accuracy is detected, it will go to the next sign
    /// </summary>
    /// <returns></returns>
    IEnumerator UpdateFeedback()
    {
        while (true)
        {
            if (getSignCallback != null && predictSignCallback != null)
            {

                // Get current sign
                string currentSign = getSignCallback();
                // Get the predicted sign
                if (signPredictor != null && signPredictor.learnableProbabilities != null &&
                    currentSign != null && signPredictor.learnableProbabilities.ContainsKey(currentSign))
                {    
                    float accuracy = signPredictor.learnableProbabilities[currentSign];
                    if (feedbackText != null && feedbackProgressImage != null){
                        if (accuracy > 0.98)
                        {
                            // TODO: fix emojis
                            feedbackText.text = "✨ Perfect ✨";
                            Color col = new Color(0xff / 255.0f, 0xcc / 255.0f, 0x00 / 255.0f);
                            feedbackText.color = col;
                            feedbackProgressImage.color = col;
                        }
                        else if (accuracy > 0.95)
                        {
                            feedbackText.text = "Super!";
                            Color col = new Color(0x00 / 255.0f, 0xff / 255.0f, 0xcc / 255.0f);
                            feedbackText.color = col;
                            feedbackProgressImage.color = col;
                        }
                        else if (accuracy > 0.90)
                        {
                            feedbackText.text = "Goed";
                            feedbackText.color = Color.green;
                            feedbackProgressImage.color = Color.green;
                        }
                        else if (accuracy > 0.80)
                        {
                            feedbackText.text = "Bijna...";
                            Color col = new Color(0xff / 255.0f, 0x66 / 255.0f, 0x00 / 255.0f);
                            feedbackText.color = col;
                            feedbackProgressImage.color = col;
                        }
                        else
                        {
                            feedbackText.text = "Detecteren...";
                            feedbackText.color = Color.red;
                            feedbackProgressImage.color = Color.red;
                        }

                        float oldValue = feedbackProgress.value;
                        // use an exponential scale 
                        float newValue = Mathf.Exp(4 * (accuracy - 1.0f));
                        feedbackProgress.gameObject.Tween("FeedbackUpdate", oldValue, newValue, 0.2f, TweenScaleFunctions.CubicEaseInOut, (t) =>
                        {
                            if (feedbackProgress != null)
                            {
                                feedbackProgress.value = t.CurrentValue;
                            }
                        });
                    }

                    // Check whether (in)correct sign has high accuracy
                    foreach (var kv in signPredictor.learnableProbabilities)
                    {
                        if (kv.Value > 0.90)
                        {
                            predictedSign = kv.Key;
                            // Correct sign
                            if (predictedSign == currentSign)
                            {
                                yield return new WaitForSeconds(1.0f);
                                predictSignCallback(predictedSign);
                                timer = DateTime.Now;
                                predictedSign = null;
                                previousIncorrectSign = null;
                            }
                            // Incorrect sign
                            else
                            {
                                if (previousIncorrectSign != predictedSign)
                                {
                                    timer = DateTime.Now;
                                    previousIncorrectSign = predictedSign;
                                }
                                else if (DateTime.Now - timer > TimeSpan.FromSeconds(2.0f))
                                {
                                    predictSignCallback(predictedSign);
                                    timer = DateTime.Now;
                                    predictedSign = null;
                                    previousIncorrectSign = null;
                                }
                            }
                            break;
                        }
                    }
                }
                else if(feedbackProgress != null)
                {
                    
                    feedbackProgress.value = 0.0f;
                }
            }

            // Wait for 200 milliseconds before updating the scale again
            yield return new WaitForSeconds(0.2f);
        }
    }
}