WeSign/unity-application
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 6 Wiki Activity

Resolve WES-131-Feedback-REfactor

Browse Source
This commit is contained in:
Jerome Coudron
2023-04-02 12:27:59 +00:00
committed by Dries Van Schuylenbergh
parent b955d2164c
commit a808e73a29
27 changed files with 663 additions and 560 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
Assets/MediaPipeUnity/Interfaces.meta Normal file
View File

@@ -0,0 +1,8 @@
fileFormatVersion: 2
guid: 73c615986873dc246893879daf74c05d
folderAsset: yes
DefaultImporter:
externalObjects: {}
userData:
assetBundleName:
assetBundleVariant:

14
Assets/MediaPipeUnity/Interfaces/Listener.cs Normal file
View File

@@ -0,0 +1,14 @@
using System.Collections;
using System.Collections.Generic;
using UnityEngine;
/// <summary>
/// Listener interface with an IEnumerator as its processing-function.
/// </summary>
public interface Listener
{
/// <summary>
/// The function that is called by the publisher.
/// </summary>
/// <returns></returns>
public IEnumerator ProcessIncomingCall();
}

2
Assets/MediaPipeUnity/Scripts/Feedback.cs.meta → Assets/MediaPipeUnity/Interfaces/Listener.cs.meta
View File

@@ -1,5 +1,5 @@
fileFormatVersion: 2
guid: 44e682a32ee15cc489bf50f3a06f717b
guid: e4c1da9896d9ba2449549a016b5fd15e
MonoImporter:
externalObjects: {}
serializedVersion: 2

14
Assets/MediaPipeUnity/Interfaces/SignPredictorInterfaces.asmdef Normal file
View File

@@ -0,0 +1,14 @@
{
"name": "SignPredictorInterfaces",
"rootNamespace": "",
"references": [],
"includePlatforms": [],
"excludePlatforms": [],
"allowUnsafeCode": false,
"overrideReferences": false,
"precompiledReferences": [],
"autoReferenced": true,
"defineConstraints": [],
"versionDefines": [],
"noEngineReferences": false
}

7
Assets/MediaPipeUnity/Interfaces/SignPredictorInterfaces.asmdef.meta Normal file
View File

@@ -0,0 +1,7 @@
fileFormatVersion: 2
guid: f55a02e98b01bc849b30d9650ccd8f15
AssemblyDefinitionImporter:
externalObjects: {}
userData:
assetBundleName:
assetBundleVariant:

17
Assets/MediaPipeUnity/Prefabs/Feedback.prefab
View File

@@ -9,7 +9,6 @@ GameObject:
serializedVersion: 6
m_Component:
- component: {fileID: 4318122119930585316}
- component: {fileID: 4318122119930585317}
m_Layer: 5
m_Name: Feedback
m_TagString: Untagged
@@ -39,22 +38,6 @@ RectTransform:
m_AnchoredPosition: {x: 0, y: 200}
m_SizeDelta: {x: 500, y: 150}
m_Pivot: {x: 0.5, y: 0}
--- !u!114 &4318122119930585317
MonoBehaviour:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 4318122119930585319}
m_Enabled: 1
m_EditorHideFlags: 0
m_Script: {fileID: 11500000, guid: 44e682a32ee15cc489bf50f3a06f717b, type: 3}
m_Name:
m_EditorClassIdentifier:
feedbackText: {fileID: 4318122120222767928}
feedbackProgress: {fileID: 4318122119968934242}
feedbackProgressImage: {fileID: 4318122120334233319}
signPredictor: {fileID: 0}
--- !u!1 &4318122119968934244
GameObject:
m_ObjectHideFlags: 0

2
Assets/MediaPipeUnity/ScriptableObjects/ModelList.asset
View File

@@ -17,4 +17,4 @@ MonoBehaviour:
- index: 0
model: {fileID: 5022602860645237092, guid: e6d85df707405ad4f97c23b07227ee99, type: 3}
- index: 1
model: {fileID: 5022602860645237092, guid: e6d85df707405ad4f97c23b07227ee99, type: 3}
model: {fileID: 0}

40
Assets/MediaPipeUnity/Scripts/AbstractFeedback.cs Normal file
View File

@@ -0,0 +1,40 @@
using DigitalRuby.Tween;
using Mediapipe.Unity.Tutorial;
using System;
using System.Collections;
using TMPro;
using UnityEngine;
using UnityEngine.Events;
using UnityEngine.UI;
/// <summary>
/// Class to display feedback during a course
/// </summary>
public abstract class AbstractFeedback : MonoBehaviour, Listener
{
/// <summary>
/// Reference to the sign predictor
/// </summary>
public SignPredictor signPredictor;
/// <summary>
/// The function that is called by the publisher on all its listeners
/// </summary>
/// <returns></returns>
public IEnumerator ProcessIncomingCall()
{
yield return StartCoroutine(UpdateFeedback());
}
/// <summary>
/// A function to add yourself as listener to the signPredictor you are holding
/// </summary>
public void AddSelfAsListener()
{
signPredictor.listeners.Add(this);
}
/// <summary>
/// The function that holds the logic to process the new probabilities of the signPredictor
/// </summary>
/// <returns></returns>
protected abstract IEnumerator UpdateFeedback();
}

11
Assets/MediaPipeUnity/Scripts/AbstractFeedback.cs.meta Normal file
View File

@@ -0,0 +1,11 @@
fileFormatVersion: 2
guid: 7b5ac794337a54143a6e3077483d96c9
MonoImporter:
externalObjects: {}
serializedVersion: 2
defaultReferences: []
executionOrder: 0
icon: {instanceID: 0}
userData:
assetBundleName:
assetBundleVariant:

182
Assets/MediaPipeUnity/Scripts/Feedback.cs
View File

@@ -1,182 +0,0 @@
using DigitalRuby.Tween;
using Mediapipe.Unity.Tutorial;
using System;
using System.Collections;
using TMPro;
using UnityEngine;
using UnityEngine.Events;
using UnityEngine.UI;
/// <summary>
/// Class to display feedback during a course
/// </summary>
public class Feedback : MonoBehaviour
{
/// <summary>
/// Reference to the feedback field
/// </summary>
public TMP_Text feedbackText;
/// <summary>
/// Reference to the progress bar
/// </summary>
public Slider feedbackProgress;
/// <summary>
/// Reference to the progress bar image, so we can add fancy colors
/// </summary>
public Image feedbackProgressImage;
/// <summary>
/// Reference to the sign predictor
/// </summary>
public SignPredictor signPredictor;
/// <summary>
/// Callback for getting the correct sign
/// </summary>
public Func<string> getSignCallback;
/// <summary>
/// Callback to initiate the next sign
/// </summary>
public UnityAction<string> predictSignCallback;
/// <summary>
/// Timer to keep track of how long a incorrect sign is performed
/// </summary>
private DateTime timer;
/// <summary>
/// Current predicted sign
/// </summary>
private string predictedSign = null;
/// <summary>
/// Previous incorrect sign, so we can keep track whether the user is wrong or the user is still changing signs
/// </summary>
private string previousIncorrectSign = null;
/// <summary>
/// Start is called before the first frame update
/// </summary>
void Start()
{
// Start the coroutine to update the scale every 200 milliseconds
StartCoroutine(UpdateFeedback());
}
/// <summary>
/// UpdateScale updates the progress bar every 200ms, updated the feedback text, and progress bar color
/// If a high enough accuracy is detected, it will go to the next sign
/// </summary>
/// <returns></returns>
IEnumerator UpdateFeedback()
{
while (true)
{
if (getSignCallback != null && predictSignCallback != null)
{
// Get current sign
string currentSign = getSignCallback();
// Get the predicted sign
if (signPredictor != null && signPredictor.learnableProbabilities != null &&
currentSign != null && signPredictor.learnableProbabilities.ContainsKey(currentSign))
{
float accuracy = signPredictor.learnableProbabilities[currentSign];
if (feedbackText != null && feedbackProgressImage != null){
if (accuracy > 0.98)
{
// TODO: fix emojis
feedbackText.text = "✨ Perfect ✨";
Color col = new Color(0xff / 255.0f, 0xcc / 255.0f, 0x00 / 255.0f);
feedbackText.color = col;
feedbackProgressImage.color = col;
}
else if (accuracy > 0.95)
{
feedbackText.text = "Super!";
Color col = new Color(0x00 / 255.0f, 0xff / 255.0f, 0xcc / 255.0f);
feedbackText.color = col;
feedbackProgressImage.color = col;
}
else if (accuracy > 0.90)
{
feedbackText.text = "Goed";
feedbackText.color = Color.green;
feedbackProgressImage.color = Color.green;
}
else if (accuracy > 0.80)
{
feedbackText.text = "Bijna...";
Color col = new Color(0xff / 255.0f, 0x66 / 255.0f, 0x00 / 255.0f);
feedbackText.color = col;
feedbackProgressImage.color = col;
}
else
{
feedbackText.text = "Detecteren...";
feedbackText.color = Color.red;
feedbackProgressImage.color = Color.red;
}
float oldValue = feedbackProgress.value;
// use an exponential scale
float newValue = Mathf.Exp(4 * (accuracy - 1.0f));
feedbackProgress.gameObject.Tween("FeedbackUpdate", oldValue, newValue, 0.2f, TweenScaleFunctions.CubicEaseInOut, (t) =>
{
if (feedbackProgress != null)
{
feedbackProgress.value = t.CurrentValue;
}
});
}
// Check whether (in)correct sign has high accuracy
foreach (var kv in signPredictor.learnableProbabilities)
{
if (kv.Value > 0.90)
{
predictedSign = kv.Key;
// Correct sign
if (predictedSign == currentSign)
{
yield return new WaitForSeconds(1.0f);
predictSignCallback(predictedSign);
timer = DateTime.Now;
predictedSign = null;
previousIncorrectSign = null;
}
// Incorrect sign
else
{
if (previousIncorrectSign != predictedSign)
{
timer = DateTime.Now;
previousIncorrectSign = predictedSign;
}
else if (DateTime.Now - timer > TimeSpan.FromSeconds(2.0f))
{
predictSignCallback(predictedSign);
timer = DateTime.Now;
predictedSign = null;
previousIncorrectSign = null;
}
}
break;
}
}
}
else if(feedbackProgress != null)
{
feedbackProgress.value = 0.0f;
}
}
// Wait for 200 milliseconds before updating the scale again
yield return new WaitForSeconds(0.2f);
}
}
}

3
Assets/MediaPipeUnity/Scripts/SignPredictor.asmdef
View File

@@ -7,7 +7,8 @@
"GUID:04c4d86a70aa56c55a78c61f1ab1a56d",
"GUID:edc93f477bb73a743a97d6882ed330b3",
"GUID:58e104b97fb3752438ada2902a36dcbf",
"GUID:7f2d0ee6dd21e1d4eb25b71b7a749d25"
"GUID:7f2d0ee6dd21e1d4eb25b71b7a749d25",
"GUID:f55a02e98b01bc849b30d9650ccd8f15"
],
"includePlatforms": [],
"excludePlatforms": [],

103
Assets/MediaPipeUnity/Scripts/SignPredictor.cs
View File

@@ -141,6 +141,8 @@ namespace Mediapipe.Unity.Tutorial
/// </summary>
private Tensor inputTensor;
public List<Listener> listeners = new List<Listener>();
/// <summary>
/// Google Mediapipe setup & run
/// </summary>
@@ -159,6 +161,7 @@ namespace Mediapipe.Unity.Tutorial
webcamTexture.Play();
yield return new WaitUntil(() => webcamTexture.width > 16);
// Set webcam aspect ratio
@@ -167,63 +170,63 @@ namespace Mediapipe.Unity.Tutorial
float webcamAspect = (float)webcamTexture.width / (float)webcamTexture.height;
screen.rectTransform.sizeDelta = new Vector2(screen.rectTransform.sizeDelta.y * webcamAspect, (screen.rectTransform.sizeDelta.y));
screen.texture = webcamTexture;
if(screen2 != null)
if (screen2 != null)
{
screen2.rectTransform.sizeDelta = new Vector2(screen2.rectTransform.sizeDelta.y * webcamAspect, (screen2.rectTransform.sizeDelta.y));
}
// TODO this method is kinda meh you should use
inputTexture = new Texture2D(width, height, TextureFormat.RGBA32, false);
pixelData = new Color32[width * height];
if (!resourceManagerIsInitialized)
if (modelList.GetCurrentModel() != null)
{
resourceManager = new StreamingAssetsResourceManager();
yield return resourceManager.PrepareAssetAsync("pose_detection.bytes");
yield return resourceManager.PrepareAssetAsync("pose_landmark_full.bytes");
yield return resourceManager.PrepareAssetAsync("face_landmark.bytes");
yield return resourceManager.PrepareAssetAsync("hand_landmark_full.bytes");
yield return resourceManager.PrepareAssetAsync("face_detection_short_range.bytes");
yield return resourceManager.PrepareAssetAsync("hand_recrop.bytes");
yield return resourceManager.PrepareAssetAsync("handedness.txt");
resourceManagerIsInitialized = true;
// TODO this method is kinda meh you should use
inputTexture = new Texture2D(width, height, TextureFormat.RGBA32, false);
pixelData = new Color32[width * height];
if (!resourceManagerIsInitialized)
{
resourceManager = new StreamingAssetsResourceManager();
yield return resourceManager.PrepareAssetAsync("pose_detection.bytes");
yield return resourceManager.PrepareAssetAsync("pose_landmark_full.bytes");
yield return resourceManager.PrepareAssetAsync("face_landmark.bytes");
yield return resourceManager.PrepareAssetAsync("hand_landmark_full.bytes");
yield return resourceManager.PrepareAssetAsync("face_detection_short_range.bytes");
yield return resourceManager.PrepareAssetAsync("hand_recrop.bytes");
yield return resourceManager.PrepareAssetAsync("handedness.txt");
resourceManagerIsInitialized = true;
}
stopwatch = new Stopwatch();
// Setting up the graph
graph = new CalculatorGraph(configAsset.text);
posestream = new OutputStream<NormalizedLandmarkListPacket, NormalizedLandmarkList>(graph, "pose_landmarks", "pose_landmarks_presence");
leftstream = new OutputStream<NormalizedLandmarkListPacket, NormalizedLandmarkList>(graph, "left_hand_landmarks", "left_hand_landmarks_presence");
rightstream = new OutputStream<NormalizedLandmarkListPacket, NormalizedLandmarkList>(graph, "right_hand_landmarks", "right_hand_landmarks_presence");
posestream.StartPolling().AssertOk();
leftstream.StartPolling().AssertOk();
rightstream.StartPolling().AssertOk();
graph.StartRun().AssertOk();
stopwatch.Start();
keypointManager = new KeypointManager(modelInfoFile);
// check if model exists at path
//var model = ModelLoader.Load(Resources.Load<NNModel>("Models/Fingerspelling/model_A-L"));
worker = modelList.GetCurrentModel().CreateWorker();
StartCoroutine(SignRecognitionCoroutine());
StartCoroutine(MediapipeCoroutine());
}
stopwatch = new Stopwatch();
// Setting up the graph
graph = new CalculatorGraph(configAsset.text);
posestream = new OutputStream<NormalizedLandmarkListPacket, NormalizedLandmarkList>(graph, "pose_landmarks", "pose_landmarks_presence");
leftstream = new OutputStream<NormalizedLandmarkListPacket, NormalizedLandmarkList>(graph, "left_hand_landmarks", "left_hand_landmarks_presence");
rightstream = new OutputStream<NormalizedLandmarkListPacket, NormalizedLandmarkList>(graph, "right_hand_landmarks", "right_hand_landmarks_presence");
posestream.StartPolling().AssertOk();
leftstream.StartPolling().AssertOk();
rightstream.StartPolling().AssertOk();
graph.StartRun().AssertOk();
stopwatch.Start();
keypointManager = new KeypointManager(modelInfoFile);
// check if model exists at path
//var model = ModelLoader.Load(Resources.Load<NNModel>("Models/Fingerspelling/model_A-L"));
worker = modelList.GetCurrentModel().CreateWorker();
StartCoroutine(SignRecognitionCoroutine());
StartCoroutine(MediapipeCoroutine());
}
public void ChangeModel(ModelIndex index)
/// <summary>
/// Called at the start of course/Minigame, will set the model before the start of SIgnPredictor is called.
/// </summary>
/// <param name="index">The index of the model to be used</param>
public void SetModel(ModelIndex index)
{
this.modelList.SetCurrentModel(index);
// If a worker already existed, we throw it out
worker?.Dispose();
// Add a new worker for the new model
worker = modelList.GetCurrentModel().CreateWorker();
}
/// <summary>
@@ -325,6 +328,10 @@ namespace Mediapipe.Unity.Tutorial
learnableProbabilities.Add(((char)(i + 65)).ToString(), softmaxedOutput2[i]);
}
//Debug.Log($"prob = [{learnableProbabilities.Aggregate(" ", (t, kv) => $"{t}{kv.Key}:{kv.Value} ")}]");
foreach(Listener listener in listeners)
{
yield return listener.ProcessIncomingCall();
}
}
else
{