WeSign/unity-application
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 6 Wiki Activity

Resolve WES-97 "Integrate signpredictor in spellingbee"

Browse Source
This commit is contained in:
Dries Van Schuylenbergh
2023-03-19 17:37:50 +00:00
committed by Lukas Van Rossem
parent f827c29d3a
commit 3abc24a39c
72 changed files with 3169 additions and 1886 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
Assets/Courses/Scripts/CourseScripts.asmdef
View File

@@ -6,7 +6,7 @@
"AccountsScripts",
"InterfacesScripts",
"SignPredictor",
"Tween"
"Unity.Barracuda"
],
"includePlatforms": [],
"excludePlatforms": [],

123
Assets/Courses/Scripts/Feedback.cs
View File

@@ -1,123 +0,0 @@
//using Mediapipe.Unity.Tutorial;
using Mediapipe.Unity.Tutorial;
using System.Collections;
using TMPro;
using UnityEngine;
using UnityEngine.UI;
// for your own scripts make sure to add the following line:
using DigitalRuby.Tween;
using UnityEngine.SceneManagement;
namespace Assets.Courses.Scripts
{
public class Feedback : MonoBehaviour
{
/// <summary>
/// Reference to the feedback field
/// </summary>
public TMP_Text feedback;
/// <summary>
/// Reference to the sign predictor
/// </summary>
public Wesign_extractor signPredictor;
/// <summary>
/// Reference to the TemplateCourse
/// </summary>
public TemplateCourse templateCourse;
/// <summary>
/// Reference to the progress bar
/// </summary>
public GameObject progress;
/// <summary>
/// Start is called before the first frame update
/// </summary>
void Start()
{
// Start the coroutine to update the scale every 200 milliseconds
StartCoroutine(UpdateFeedback());
}
/// <summary>
/// UpdateScale updates the progress bar every 200ms, updated the feedback text, and progress bar color
/// If a high enough accuracy is detected, it will go to the next sign
/// </summary>
/// <returns></returns>
IEnumerator UpdateFeedback()
{
while (true)
{
// Get current sign
char currentSign = (char)(65 + templateCourse.GetWordIndex());
//Debug.Log(currentSign);
// Get the predicted sign
if (signPredictor != null && signPredictor.letterProbabilities != null && signPredictor.letterProbabilities.ContainsKey(currentSign))
{
int accuracy = (int)(signPredictor.letterProbabilities[currentSign] * 100);
if (accuracy > 98)
{
feedback.text = "Perfect!!!";
feedback.color = Color.green;
progress.GetComponent<Image>().color = Color.green;
}
else if (accuracy > 95)
{
feedback.text = "Super!";
feedback.color = Color.green;
progress.GetComponent<Image>().color = Color.green;
}
else if (accuracy > 90)
{
feedback.text = "Goed";
feedback.color = Color.green;
progress.GetComponent<Image>().color = Color.green;
}
else if (accuracy > 80)
{
feedback.text = "Bijna...";
feedback.color = new Color(0xFF, 0xE5, 0x00);
progress.GetComponent<Image>().color = new Color(0xFF, 0xE5, 0x00);
}
else
{
feedback.text = "Detecteren ...";
feedback.color = Color.red;
progress.GetComponent<Image>().color = Color.red;
}
// use an exponential scale
float newScale = Mathf.Exp(((float)accuracy / 24.5f) - 4);
Vector3 newScaleVector = new Vector3(newScale,
progress.transform.localScale.y,
progress.transform.localScale.z);
System.Action<ITween<Vector3>> updateProgressScale = (t) =>
{
if (progress != null)
{
progress.transform.localScale = t.CurrentValue;
}
};
progress.Tween("ScaleProgress", progress.transform.localScale, newScaleVector, 0.2f, TweenScaleFunctions.CubicEaseInOut, updateProgressScale);
if (accuracy > 90)
{
// Wait and go to next sign
yield return new WaitForSeconds(1);
templateCourse.NextSign();
}
}
else
{
progress.transform.localScale = new Vector3(0f, progress.transform.localScale.y, progress.transform.localScale.z);
//Debug.Log("doesn't contain A");
}
// Wait for 200 milliseconds before updating the scale again
yield return new WaitForSeconds(0.2f);
}
}
}
}

11
Assets/Courses/Scripts/Feedback.cs.meta
View File

@@ -1,11 +0,0 @@
fileFormatVersion: 2
guid: 44e682a32ee15cc489bf50f3a06f717b
MonoImporter:
externalObjects: {}
serializedVersion: 2
defaultReferences: []
executionOrder: 0
icon: {instanceID: 0}
userData:
assetBundleName:
assetBundleVariant:

150
Assets/Courses/Scripts/TemplateCourse.cs
View File

@@ -1,5 +1,6 @@
using System;
using TMPro;
using Unity.Barracuda;
using UnityEngine;
using UnityEngine.UI;
using UnityEngine.Video;
@@ -9,36 +10,17 @@ using UnityEngine.Video;
/// </summary>
public class TemplateCourse : MonoBehaviour
{
/// <summary>
/// Index to indicate which camera is being used
/// </summary>
private int camdex = 0;
/// <summary>
/// This texture is used as an intermidiary between the camera output and the display image
/// </summary>
private WebCamTexture tex;
/// <summary>
/// Reference to the RawImage that will display the video
/// </summary>
public RawImage webcamDisplay;
/// <summary>
/// Reference to the button that is currently used to test the feedback-display
/// </summary>
public Button feedback;
/// <summary>
/// This is a reference to the textfield that holds the part of the feedback-window that will change: bad/good/excellent
/// </summary>
public TMP_Text dynamic;
// vvv TEMPORARY STUFF vvv
public NNModel previewModel;
public GameObject feedbackProgressBar;
public GameObject previewMessage;
// ^^^ TEMPORARY STUFF ^^^
/// <summary>
/// Reference to instructional video player
/// </summary>
public VideoPlayer player;
/// <summary>
/// Reference to pause button
/// </summary>
@@ -120,6 +102,11 @@ public class TemplateCourse : MonoBehaviour
/// </summary>
public TMP_Text timeSpent;
/// <summary>
/// Reference to the feedback script on the Feedback prefab
/// </summary>
public Feedback feedback;
/// <summary>
/// This function is called when the script is initialised.
/// It inactivatis the popup, finds a webcam to use and links it via the WebcamTexture to the display RawImage.
@@ -129,21 +116,17 @@ public class TemplateCourse : MonoBehaviour
/// </summary>
void Awake()
{
// Setting up Webcam
// feedbackPopup.SetActive(false);
//if (WebCamTexture.devices.Length > 0)
//{
// WebCamDevice device = WebCamTexture.devices[camdex];
// tex = new WebCamTexture(device.name);
// webcamDisplay.texture = tex;
// tex.Play();
//}
// Setting up course
course = courselist.courses[courselist.currentCourseIndex];
feedback.signPredictor.model = course.theme.model;
maxWords = course.theme.learnables.Count;
// vvv TEMPORARY STUFF vvv
feedbackProgressBar.SetActive(course.theme.model != null);
previewMessage.SetActive(course.theme.model == null);
feedback.signPredictor.model = previewModel;
// ^^^ TEMPORARY STUFF ^^^
// Create entry in current user for keeping track of progress
user = userList.GetCurrentUser();
progress = user.GetCourseProgress(course.index);
@@ -161,7 +144,7 @@ public class TemplateCourse : MonoBehaviour
// Setup UI
button.image.sprite = pauseSprite;
title.text = course.name;
title.text = course.title;
NextVideo();
NextImage();
@@ -169,6 +152,23 @@ public class TemplateCourse : MonoBehaviour
ResultPanel.SetActive(false);
// Set the startTime
startMoment = DateTime.Now;
// Set callbacks
feedback.getSignCallback = () =>
{
if (currentWordIndex < course.theme.learnables.Count)
{
return course.theme.learnables[currentWordIndex].name;
}
return null;
};
feedback.predictSignCallback = (sign) =>
{
if (sign == course.theme.learnables[currentWordIndex].name)
{
NextSign();
}
};
}
/// <summary>
@@ -269,80 +269,4 @@ public class TemplateCourse : MonoBehaviour
progress.AddOrUpdate<float>("courseProgress", 1f);
userList.Save();
}
/// <summary>
/// So long as there are cameras to use, you swap the camera you are using to another in the list.
/// </summary>
public void SwapCam()
{
if (WebCamTexture.devices.Length > 0)
{
// Stop the old camera
// If there was no camera playing before, then you dont have to reset the texture, as it wasn't assigned in the first place.
if (tex.isPlaying)
{
webcamDisplay.texture = null;
tex.Stop();
tex = null;
}
// Find the new camera
camdex += 1;
camdex %= WebCamTexture.devices.Length;
// Start the new camera
WebCamDevice device = WebCamTexture.devices[camdex];
tex = new WebCamTexture(device.name);
webcamDisplay.texture = tex;
tex.Play();
}
}
/// <summary>
/// The normal sceneChanger cannot be used here since the camera also needs to be stopped.
/// This extra functionality is implemented in this function
/// </summary>
/// <param name="sceneName"> The path for the scene you want to travel to, assuming root-directory is Assets</param>
public void Back()
{
//webcamDisplay.texture = null;
//tex.Stop();
//tex = null;
SystemController.GetInstance().BackToPreviousScene();
}
/// <summary>
/// This function toggles between inactivity and activity for the popup panel.
/// This will be changed later when the model gets integrated, probably being timed to dissapear.
/// </summary>
//public void ShowFeedback()
//{
// if (feedbackPopup.activeSelf)
// {
// dynamic.text = "";
// feedbackPopup.SetActive(false);
// return;
// }
// double index = UnityEngine.Random.value;
// if (index < 0.5)
// {
// dynamic.text = "Poor";
// }
// else if (index > 0.8)
// {
// dynamic.text = "Excellent";
// }
// else
// {
// dynamic.text = "Good";
// }
// feedbackPopup.SetActive(true);
//}
// Get currentWordIndex
public int GetWordIndex()
{
return currentWordIndex;
}
}