WeSign/unity-application
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 6 Wiki Activity

Sprint 4

Browse Source
This commit is contained in:
Jelle De Geest
2023-04-26 19:04:34 +02:00
parent 172938cec8
commit 47f8b96122
1004 changed files with 60756 additions and 117444 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
Assets/MediaPipeUnity/Scripts/AbstractFeedback.cs Normal file
View File

@@ -0,0 +1,36 @@
using System.Collections;
using UnityEngine;
/// <summary>
/// Class to display feedback during a course
/// </summary>
public abstract class AbstractFeedback : MonoBehaviour, Listener
{
/// <summary>
/// Reference to the sign predictor
/// </summary>
public SignPredictor signPredictor;
/// <summary>
/// The function that is called by the publisher on all its listeners
/// </summary>
/// <returns></returns>
public IEnumerator ProcessIncomingCall()
{
yield return StartCoroutine(UpdateFeedback());
}
/// <summary>
/// A function to add yourself as listener to the signPredictor you are holding
/// </summary>
public void AddSelfAsListener()
{
signPredictor.listeners.Add(this);
}
/// <summary>
/// The function that holds the logic to process the new probabilities of the signPredictor
/// </summary>
/// <returns></returns>
protected abstract IEnumerator UpdateFeedback();
}

2
Assets/MediaPipeUnity/Scripts/Feedback.cs.meta → Assets/MediaPipeUnity/Scripts/AbstractFeedback.cs.meta
View File

@@ -1,5 +1,5 @@
fileFormatVersion: 2
guid: 44e682a32ee15cc489bf50f3a06f717b
guid: 7b5ac794337a54143a6e3077483d96c9
MonoImporter:
externalObjects: {}
serializedVersion: 2

182
Assets/MediaPipeUnity/Scripts/Feedback.cs
View File

@@ -1,182 +0,0 @@
using DigitalRuby.Tween;
using Mediapipe.Unity.Tutorial;
using System;
using System.Collections;
using TMPro;
using UnityEngine;
using UnityEngine.Events;
using UnityEngine.UI;
/// <summary>
/// Class to display feedback during a course
/// </summary>
public class Feedback : MonoBehaviour
{
/// <summary>
/// Reference to the feedback field
/// </summary>
public TMP_Text feedbackText;
/// <summary>
/// Reference to the progress bar
/// </summary>
public Slider feedbackProgress;
/// <summary>
/// Reference to the progress bar image, so we can add fancy colors
/// </summary>
public Image feedbackProgressImage;
/// <summary>
/// Reference to the sign predictor
/// </summary>
public SignPredictor signPredictor;
/// <summary>
/// Callback for getting the correct sign
/// </summary>
public Func<string> getSignCallback;
/// <summary>
/// Callback to initiate the next sign
/// </summary>
public UnityAction<string> predictSignCallback;
/// <summary>
/// Timer to keep track of how long a incorrect sign is performed
/// </summary>
private DateTime timer;
/// <summary>
/// Current predicted sign
/// </summary>
private string predictedSign = null;
/// <summary>
/// Previous incorrect sign, so we can keep track whether the user is wrong or the user is still changing signs
/// </summary>
private string previousIncorrectSign = null;
/// <summary>
/// Start is called before the first frame update
/// </summary>
void Start()
{
// Start the coroutine to update the scale every 200 milliseconds
StartCoroutine(UpdateFeedback());
}
/// <summary>
/// UpdateScale updates the progress bar every 200ms, updated the feedback text, and progress bar color
/// If a high enough accuracy is detected, it will go to the next sign
/// </summary>
/// <returns></returns>
IEnumerator UpdateFeedback()
{
while (true)
{
if (getSignCallback != null && predictSignCallback != null)
{
// Get current sign
string currentSign = getSignCallback();
// Get the predicted sign
if (signPredictor != null && signPredictor.learnableProbabilities != null &&
currentSign != null && signPredictor.learnableProbabilities.ContainsKey(currentSign))
{
float accuracy = signPredictor.learnableProbabilities[currentSign];
if (feedbackText != null && feedbackProgressImage != null){
if (accuracy > 0.98)
{
// TODO: fix emojis
feedbackText.text = "✨ Perfect ✨";
Color col = new Color(0xff / 255.0f, 0xcc / 255.0f, 0x00 / 255.0f);
feedbackText.color = col;
feedbackProgressImage.color = col;
}
else if (accuracy > 0.95)
{
feedbackText.text = "Super!";
Color col = new Color(0x00 / 255.0f, 0xff / 255.0f, 0xcc / 255.0f);
feedbackText.color = col;
feedbackProgressImage.color = col;
}
else if (accuracy > 0.90)
{
feedbackText.text = "Goed";
feedbackText.color = Color.green;
feedbackProgressImage.color = Color.green;
}
else if (accuracy > 0.80)
{
feedbackText.text = "Bijna...";
Color col = new Color(0xff / 255.0f, 0x66 / 255.0f, 0x00 / 255.0f);
feedbackText.color = col;
feedbackProgressImage.color = col;
}
else
{
feedbackText.text = "Detecteren...";
feedbackText.color = Color.red;
feedbackProgressImage.color = Color.red;
}
float oldValue = feedbackProgress.value;
// use an exponential scale
float newValue = Mathf.Exp(4 * (accuracy - 1.0f));
feedbackProgress.gameObject.Tween("FeedbackUpdate", oldValue, newValue, 0.2f, TweenScaleFunctions.CubicEaseInOut, (t) =>
{
if (feedbackProgress != null)
{
feedbackProgress.value = t.CurrentValue;
}
});
}
// Check whether (in)correct sign has high accuracy
foreach (var kv in signPredictor.learnableProbabilities)
{
if (kv.Value > 0.90)
{
predictedSign = kv.Key;
// Correct sign
if (predictedSign == currentSign)
{
yield return new WaitForSeconds(1.0f);
predictSignCallback(predictedSign);
timer = DateTime.Now;
predictedSign = null;
previousIncorrectSign = null;
}
// Incorrect sign
else
{
if (previousIncorrectSign != predictedSign)
{
timer = DateTime.Now;
previousIncorrectSign = predictedSign;
}
else if (DateTime.Now - timer > TimeSpan.FromSeconds(2.0f))
{
predictSignCallback(predictedSign);
timer = DateTime.Now;
predictedSign = null;
previousIncorrectSign = null;
}
}
break;
}
}
}
else if(feedbackProgress != null)
{
feedbackProgress.value = 0.0f;
}
}
// Wait for 200 milliseconds before updating the scale again
yield return new WaitForSeconds(0.2f);
}
}
}

6
Assets/MediaPipeUnity/Scripts/SignPredictor.asmdef
View File

@@ -3,10 +3,12 @@
"rootNamespace": "",
"references": [
"GUID:6055be8ebefd69e48b49212b09b47b2f",
"GUID:5c2b5ba89f9e74e418232e154bc5cc7a",
"GUID:04c4d86a70aa56c55a78c61f1ab1a56d",
"GUID:edc93f477bb73a743a97d6882ed330b3",
"GUID:58e104b97fb3752438ada2902a36dcbf"
"GUID:58e104b97fb3752438ada2902a36dcbf",
"GUID:7f2d0ee6dd21e1d4eb25b71b7a749d25",
"GUID:f55a02e98b01bc849b30d9650ccd8f15",
"GUID:d23f64cfd3b314bb4a18a8284c99bf5e"
],
"includePlatforms": [],
"excludePlatforms": [],

982
Assets/MediaPipeUnity/Scripts/SignPredictor.cs
View File

File diff suppressed because it is too large Load Diff