Resolve WES-131-Feedback-REfactor
This commit is contained in:
committed by
Dries Van Schuylenbergh
parent
b955d2164c
commit
a808e73a29
@@ -8,7 +8,8 @@
|
||||
"CommonScripts",
|
||||
"InterfacesScripts",
|
||||
"Unity.TextMeshPro",
|
||||
"AccountsScripts"
|
||||
"AccountsScripts",
|
||||
"SignPredictor"
|
||||
],
|
||||
"includePlatforms": [],
|
||||
"excludePlatforms": [],
|
||||
|
||||
@@ -5,7 +5,6 @@
|
||||
"GUID:6055be8ebefd69e48b49212b09b47b2f",
|
||||
"GUID:63c63e721f65ebb7d871cb9ef49f4752",
|
||||
"GUID:1631ed2680c61245b8211d943c1639a8",
|
||||
"GUID:5c2b5ba89f9e74e418232e154bc5cc7a",
|
||||
"GUID:7f2d0ee6dd21e1d4eb25b71b7a749d25"
|
||||
],
|
||||
"includePlatforms": [],
|
||||
|
||||
@@ -1273,7 +1273,6 @@ MonoBehaviour:
|
||||
m_Name:
|
||||
m_EditorClassIdentifier:
|
||||
modelList: {fileID: 11400000, guid: 39516e4e6e56f0f4f80647d9c4d8034c, type: 2}
|
||||
model: {fileID: 5022602860645237092, guid: e6d85df707405ad4f97c23b07227ee99, type: 3}
|
||||
modelInfoFile: {fileID: 4900000, guid: fb8b51022bdcd654a9f29c054832a1b5, type: 3}
|
||||
configAsset: {fileID: 4900000, guid: 6288c43cdca97374782dac1ea87aa029, type: 3}
|
||||
screen: {fileID: 378145456}
|
||||
@@ -1529,6 +1528,7 @@ MonoBehaviour:
|
||||
m_Script: {fileID: 11500000, guid: 6b3f784c065813a4a8364b1299284816, type: 3}
|
||||
m_Name:
|
||||
m_EditorClassIdentifier:
|
||||
signPredictor: {fileID: 883853268}
|
||||
previewModel: {fileID: 5022602860645237092, guid: e6d85df707405ad4f97c23b07227ee99, type: 3}
|
||||
feedbackProgressBar: {fileID: 4318122121437849759}
|
||||
previewMessage: {fileID: 2070775951}
|
||||
@@ -1544,7 +1544,9 @@ MonoBehaviour:
|
||||
ResultsDecription: {fileID: 100123246}
|
||||
CoursesButton: {fileID: 839294691}
|
||||
timeSpent: {fileID: 77614869}
|
||||
feedback: {fileID: 1714882683}
|
||||
feedbackText: {fileID: 4318122121437849762}
|
||||
feedbackProgress: {fileID: 4318122121437849761}
|
||||
feedbackProgressImage: {fileID: 4318122121437849760}
|
||||
--- !u!4 &1122267057
|
||||
Transform:
|
||||
m_ObjectHideFlags: 0
|
||||
@@ -2284,17 +2286,6 @@ RectTransform:
|
||||
m_CorrespondingSourceObject: {fileID: 4318122119930585316, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3}
|
||||
m_PrefabInstance: {fileID: 4318122121437849758}
|
||||
m_PrefabAsset: {fileID: 0}
|
||||
--- !u!114 &1714882683 stripped
|
||||
MonoBehaviour:
|
||||
m_CorrespondingSourceObject: {fileID: 4318122119930585317, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3}
|
||||
m_PrefabInstance: {fileID: 4318122121437849758}
|
||||
m_PrefabAsset: {fileID: 0}
|
||||
m_GameObject: {fileID: 4318122121437849759}
|
||||
m_Enabled: 1
|
||||
m_EditorHideFlags: 0
|
||||
m_Script: {fileID: 11500000, guid: 44e682a32ee15cc489bf50f3a06f717b, type: 3}
|
||||
m_Name:
|
||||
m_EditorClassIdentifier:
|
||||
--- !u!1 &1773033262
|
||||
GameObject:
|
||||
m_ObjectHideFlags: 0
|
||||
@@ -2897,3 +2888,36 @@ GameObject:
|
||||
m_CorrespondingSourceObject: {fileID: 4318122119930585319, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3}
|
||||
m_PrefabInstance: {fileID: 4318122121437849758}
|
||||
m_PrefabAsset: {fileID: 0}
|
||||
--- !u!114 &4318122121437849760 stripped
|
||||
MonoBehaviour:
|
||||
m_CorrespondingSourceObject: {fileID: 4318122120334233319, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3}
|
||||
m_PrefabInstance: {fileID: 4318122121437849758}
|
||||
m_PrefabAsset: {fileID: 0}
|
||||
m_GameObject: {fileID: 0}
|
||||
m_Enabled: 1
|
||||
m_EditorHideFlags: 0
|
||||
m_Script: {fileID: 11500000, guid: fe87c0e1cc204ed48ad3b37840f39efc, type: 3}
|
||||
m_Name:
|
||||
m_EditorClassIdentifier:
|
||||
--- !u!114 &4318122121437849761 stripped
|
||||
MonoBehaviour:
|
||||
m_CorrespondingSourceObject: {fileID: 4318122119968934242, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3}
|
||||
m_PrefabInstance: {fileID: 4318122121437849758}
|
||||
m_PrefabAsset: {fileID: 0}
|
||||
m_GameObject: {fileID: 0}
|
||||
m_Enabled: 1
|
||||
m_EditorHideFlags: 0
|
||||
m_Script: {fileID: 11500000, guid: 67db9e8f0e2ae9c40bc1e2b64352a6b4, type: 3}
|
||||
m_Name:
|
||||
m_EditorClassIdentifier:
|
||||
--- !u!114 &4318122121437849762 stripped
|
||||
MonoBehaviour:
|
||||
m_CorrespondingSourceObject: {fileID: 4318122120222767928, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3}
|
||||
m_PrefabInstance: {fileID: 4318122121437849758}
|
||||
m_PrefabAsset: {fileID: 0}
|
||||
m_GameObject: {fileID: 0}
|
||||
m_Enabled: 1
|
||||
m_EditorHideFlags: 0
|
||||
m_Script: {fileID: 11500000, guid: f4688fdb7df04437aeb418b961361dc5, type: 3}
|
||||
m_Name:
|
||||
m_EditorClassIdentifier:
|
||||
|
||||
@@ -5,8 +5,8 @@
|
||||
"Unity.TextMeshPro",
|
||||
"AccountsScripts",
|
||||
"InterfacesScripts",
|
||||
"SignPredictor",
|
||||
"Unity.Barracuda"
|
||||
"Tween",
|
||||
"SignPredictor"
|
||||
],
|
||||
"includePlatforms": [],
|
||||
"excludePlatforms": [],
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
using DigitalRuby.Tween;
|
||||
using System;
|
||||
using System.Collections;
|
||||
using TMPro;
|
||||
using Unity.Barracuda;
|
||||
using UnityEngine;
|
||||
using UnityEngine.UI;
|
||||
using UnityEngine.Video;
|
||||
@@ -8,13 +9,10 @@ using UnityEngine.Video;
|
||||
/// <summary>
|
||||
/// TemplateCourse scene manager
|
||||
/// </summary>
|
||||
public class CoursesController : MonoBehaviour
|
||||
public class CoursesController : AbstractFeedback
|
||||
{
|
||||
// vvv TEMPORARY STUFF vvv
|
||||
public NNModel previewModel;
|
||||
public GameObject feedbackProgressBar;
|
||||
public GameObject previewMessage;
|
||||
// ^^^ TEMPORARY STUFF ^^^
|
||||
|
||||
/// <summary>
|
||||
/// Reference to instructional video player
|
||||
@@ -112,10 +110,37 @@ public class CoursesController : MonoBehaviour
|
||||
/// </summary>
|
||||
public TMP_Text timeSpent;
|
||||
|
||||
|
||||
/// <summary>
|
||||
/// Reference to the feedback script on the Feedback prefab
|
||||
/// Reference to the feedback field
|
||||
/// </summary>
|
||||
public Feedback feedback;
|
||||
public TMP_Text feedbackText;
|
||||
|
||||
/// <summary>
|
||||
/// Reference to the progress bar
|
||||
/// </summary>
|
||||
public Slider feedbackProgress;
|
||||
|
||||
/// <summary>
|
||||
/// Reference to the progress bar image, so we can add fancy colors
|
||||
/// </summary>
|
||||
public Image feedbackProgressImage;
|
||||
|
||||
/// <summary>
|
||||
/// Timer to keep track of how long a incorrect sign is performed
|
||||
/// </summary>
|
||||
protected DateTime timer;
|
||||
|
||||
/// <summary>
|
||||
/// Current predicted sign
|
||||
/// </summary>
|
||||
protected string predictedSign = null;
|
||||
|
||||
/// <summary>
|
||||
/// Previous incorrect sign, so we can keep track whether the user is wrong or the user is still changing signs
|
||||
/// </summary>
|
||||
protected string previousIncorrectSign = null;
|
||||
|
||||
|
||||
/// <summary>
|
||||
/// This function is called when the script is initialised.
|
||||
@@ -124,19 +149,25 @@ public class CoursesController : MonoBehaviour
|
||||
/// Then it checks whether or not the User has started the course yet, to possibly create a new progress atribute for the course.
|
||||
/// Then it sets up the course-screen to display relevant information from the course-scriptable.
|
||||
/// </summary>
|
||||
void Awake()
|
||||
void Start()
|
||||
{
|
||||
StartCourseController();
|
||||
|
||||
signPredictor.SetModel(course.theme.modelIndex);
|
||||
AddSelfAsListener();
|
||||
}
|
||||
/// <summary>
|
||||
/// Holds the course-specific logic to start the controller, it is seperated to allow the course to be reset (if that would become needed)
|
||||
/// </summary>
|
||||
public void StartCourseController()
|
||||
{
|
||||
// Setting up course
|
||||
course = courselist.courses[courselist.currentCourseIndex];
|
||||
feedback.signPredictor.ChangeModel(course.theme.modelIndex);
|
||||
maxWords = course.theme.learnables.Count;
|
||||
|
||||
// vvv TEMPORARY STUFF vvv
|
||||
// Show preview messages if there is no model
|
||||
feedbackProgressBar.SetActive(course.theme.modelIndex != ModelIndex.NONE);
|
||||
previewMessage.SetActive(course.theme.modelIndex == ModelIndex.NONE);
|
||||
// Instead, the NONE-modelIndex points to Fingerspelling, which gives the same result
|
||||
//feedback.signPredictor.model = previewModel;
|
||||
// ^^^ TEMPORARY STUFF ^^^
|
||||
|
||||
// Create entry in current user for keeping track of progress
|
||||
userList.Load();
|
||||
@@ -164,23 +195,6 @@ public class CoursesController : MonoBehaviour
|
||||
ResultPanel.SetActive(false);
|
||||
// Set the startTime
|
||||
startMoment = DateTime.Now;
|
||||
|
||||
// Set callbacks
|
||||
feedback.getSignCallback = () =>
|
||||
{
|
||||
if (currentWordIndex < course.theme.learnables.Count)
|
||||
{
|
||||
return course.theme.learnables[currentWordIndex].name;
|
||||
}
|
||||
return null;
|
||||
};
|
||||
feedback.predictSignCallback = (sign) =>
|
||||
{
|
||||
if (sign == course.theme.learnables[currentWordIndex].name)
|
||||
{
|
||||
NextSign();
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -287,4 +301,104 @@ public class CoursesController : MonoBehaviour
|
||||
progress.AddOrUpdate<float>("courseProgress", 1f);
|
||||
userList.Save();
|
||||
}
|
||||
/// <summary>
|
||||
/// The updateFunction that is called when new probabilities become available
|
||||
/// </summary>
|
||||
/// <returns></returns>
|
||||
protected override IEnumerator UpdateFeedback()
|
||||
{
|
||||
// Get current sign
|
||||
string currentSign = course.theme.learnables[currentWordIndex].name;
|
||||
// Get the predicted sign
|
||||
if (signPredictor != null && signPredictor.learnableProbabilities != null &&
|
||||
currentSign != null && signPredictor.learnableProbabilities.ContainsKey(currentSign))
|
||||
{
|
||||
float accuracy = signPredictor.learnableProbabilities[currentSign];
|
||||
if (feedbackText != null && feedbackProgressImage != null)
|
||||
{
|
||||
if (accuracy > 0.90)
|
||||
{
|
||||
feedbackText.text = "Goed";
|
||||
feedbackText.color = Color.green;
|
||||
feedbackProgressImage.color = Color.green;
|
||||
}
|
||||
else if (accuracy > 0.80)
|
||||
{
|
||||
feedbackText.text = "Bijna...";
|
||||
Color col = new Color(0xff / 255.0f, 0x66 / 255.0f, 0x00 / 255.0f);
|
||||
feedbackText.color = col;
|
||||
feedbackProgressImage.color = col;
|
||||
}
|
||||
else
|
||||
{
|
||||
feedbackText.text = "Detecteren...";
|
||||
feedbackText.color = Color.red;
|
||||
feedbackProgressImage.color = Color.red;
|
||||
}
|
||||
|
||||
float oldValue = feedbackProgress.value;
|
||||
// use an exponential scale
|
||||
float newValue = Mathf.Exp(4 * (accuracy - 1.0f));
|
||||
feedbackProgress.gameObject.Tween("FeedbackUpdate", oldValue, newValue, 0.2f, TweenScaleFunctions.CubicEaseInOut, (t) =>
|
||||
{
|
||||
if (feedbackProgress != null)
|
||||
{
|
||||
feedbackProgress.value = t.CurrentValue;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Check whether (in)correct sign has high accuracy
|
||||
foreach (var kv in signPredictor.learnableProbabilities)
|
||||
{
|
||||
if (kv.Value > 0.90)
|
||||
{
|
||||
predictedSign = kv.Key;
|
||||
// Correct sign
|
||||
if (predictedSign == currentSign)
|
||||
{
|
||||
yield return new WaitForSeconds(1.0f);
|
||||
CheckEquality(predictedSign);
|
||||
timer = DateTime.Now;
|
||||
predictedSign = null;
|
||||
previousIncorrectSign = null;
|
||||
}
|
||||
// Incorrect sign
|
||||
else
|
||||
{
|
||||
if (previousIncorrectSign != predictedSign)
|
||||
{
|
||||
timer = DateTime.Now;
|
||||
previousIncorrectSign = predictedSign;
|
||||
}
|
||||
else if (DateTime.Now - timer > TimeSpan.FromSeconds(2.0f))
|
||||
{
|
||||
CheckEquality(predictedSign);
|
||||
timer = DateTime.Now;
|
||||
predictedSign = null;
|
||||
previousIncorrectSign = null;
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
else if (feedbackProgress != null)
|
||||
{
|
||||
|
||||
feedbackProgress.value = 0.0f;
|
||||
}
|
||||
|
||||
}
|
||||
/// <summary>
|
||||
/// Function to check equality between the current sign and the sign that the model predicted, if they are equal then the next sign is fetched.
|
||||
/// </summary>
|
||||
/// <param name="predicted"></param>
|
||||
private void CheckEquality(string predicted)
|
||||
{
|
||||
if (predicted == course.theme.learnables[currentWordIndex].name)
|
||||
{
|
||||
NextSign();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -38,7 +38,7 @@ RenderSettings:
|
||||
m_ReflectionIntensity: 1
|
||||
m_CustomReflection: {fileID: 0}
|
||||
m_Sun: {fileID: 0}
|
||||
m_IndirectSpecularColor: {r: 0.37311953, g: 0.38074014, b: 0.3587274, a: 1}
|
||||
m_IndirectSpecularColor: {r: 0.37311918, g: 0.3807398, b: 0.35872716, a: 1}
|
||||
m_UseRadianceAmbientProbe: 0
|
||||
--- !u!157 &3
|
||||
LightmapSettings:
|
||||
@@ -5129,6 +5129,7 @@ MonoBehaviour:
|
||||
m_Script: {fileID: 11500000, guid: 2db44635e0eb1e9429a2e6195785364d, type: 3}
|
||||
m_Name:
|
||||
m_EditorClassIdentifier:
|
||||
signPredictor: {fileID: 1991376311}
|
||||
themelist: {fileID: 11400000, guid: a247e2ce790f0f746a3bc521e6ab7d58, type: 2}
|
||||
letterPrefab: {fileID: 4639383499500021565, guid: c3e66e8957864914cb022af914df6a28, type: 3}
|
||||
letterContainer: {fileID: 1870283439}
|
||||
@@ -5153,8 +5154,10 @@ MonoBehaviour:
|
||||
Scoreboard: {fileID: 1007532375}
|
||||
EntriesGrid: {fileID: 1391137944}
|
||||
scoreboardEntry: {fileID: 9154151134820372555, guid: d4a3a228b08d61847acc6da35b44e52c, type: 3}
|
||||
feedback: {fileID: 5233312447513285388}
|
||||
gottogamebutton: {fileID: 1581633295}
|
||||
feedbackText: {fileID: 0}
|
||||
feedbackProgress: {fileID: 0}
|
||||
feedbackProgressImage: {fileID: 0}
|
||||
--- !u!1 &1678036720
|
||||
GameObject:
|
||||
m_ObjectHideFlags: 0
|
||||
@@ -6183,7 +6186,6 @@ MonoBehaviour:
|
||||
m_Name:
|
||||
m_EditorClassIdentifier:
|
||||
modelList: {fileID: 11400000, guid: 39516e4e6e56f0f4f80647d9c4d8034c, type: 2}
|
||||
model: {fileID: 5022602860645237092, guid: e6d85df707405ad4f97c23b07227ee99, type: 3}
|
||||
modelInfoFile: {fileID: 4900000, guid: fb8b51022bdcd654a9f29c054832a1b5, type: 3}
|
||||
configAsset: {fileID: 4900000, guid: 6288c43cdca97374782dac1ea87aa029, type: 3}
|
||||
screen: {fileID: 1649505745}
|
||||
|
||||
@@ -1,62 +0,0 @@
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
using UnityEngine;
|
||||
using UnityEngine.UI;
|
||||
|
||||
public class HangManWebcam : WebCam
|
||||
{
|
||||
/// <summary>
|
||||
/// The display for player 1
|
||||
/// </summary>
|
||||
public RawImage display1;
|
||||
|
||||
/// <summary>
|
||||
/// The display for player 2
|
||||
/// </summary>
|
||||
public RawImage display2;
|
||||
|
||||
/// <summary>
|
||||
/// We use a different awake, since we dont want the camera to start immediatelly
|
||||
/// </summary>
|
||||
void Awake()
|
||||
{
|
||||
WebCamDevice device = WebCamTexture.devices[camdex];
|
||||
tex = new WebCamTexture(device.name);
|
||||
display.texture = tex;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Hangman uses two different webcam_textures, we need to be able to toggle between them
|
||||
/// </summary>
|
||||
public void Switch_texture()
|
||||
{
|
||||
if(display == display1)
|
||||
{
|
||||
display = display2;
|
||||
}
|
||||
else
|
||||
{
|
||||
display = display1;
|
||||
}
|
||||
// Give the webcamTexture to the new webcam
|
||||
display.texture = tex;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Scene changing is implemented here to avoid problems with webcam
|
||||
/// </summary>
|
||||
public new void GotoThemeSelection()
|
||||
{
|
||||
//minigameList.GetIndexInMinigameList(MinigameIndex.HANGMAN);
|
||||
if (tex != null)
|
||||
{
|
||||
if (tex.isPlaying)
|
||||
{
|
||||
display.texture = null;
|
||||
tex.Stop();
|
||||
tex = null;
|
||||
}
|
||||
}
|
||||
SystemController.GetInstance().BackToPreviousScene();
|
||||
}
|
||||
}
|
||||
@@ -1,11 +1,13 @@
|
||||
using System;
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using TMPro;
|
||||
using UnityEngine;
|
||||
using UnityEngine.UI;
|
||||
using DigitalRuby.Tween;
|
||||
|
||||
public class HangmanController : MonoBehaviour
|
||||
public class HangmanController : AbstractFeedback
|
||||
{
|
||||
/// <summary>
|
||||
/// The scriptable with all the themes, will be used to select a random word for hangman.
|
||||
@@ -18,11 +20,6 @@ public class HangmanController : MonoBehaviour
|
||||
/// </summary>
|
||||
private string currentWord;
|
||||
|
||||
/// <summary>
|
||||
/// All of the words that can be used in this session
|
||||
/// </summary>
|
||||
private string[] words;
|
||||
|
||||
/// <summary>
|
||||
/// This integer holds the total amount of wrong guesses the player has made
|
||||
/// </summary>
|
||||
@@ -200,11 +197,6 @@ public class HangmanController : MonoBehaviour
|
||||
/// </summary>
|
||||
public GameObject scoreboardEntry;
|
||||
|
||||
/// <summary>
|
||||
/// Accuracy feeback object
|
||||
/// </summary>
|
||||
public Feedback feedback;
|
||||
|
||||
/// <summary>
|
||||
/// The button to go into the game
|
||||
/// </summary>
|
||||
@@ -215,8 +207,50 @@ public class HangmanController : MonoBehaviour
|
||||
/// </summary>
|
||||
private String currentsign = "";
|
||||
|
||||
// Start is called before the first frame update
|
||||
/// <summary>
|
||||
/// Reference to the feedback field
|
||||
/// </summary>
|
||||
public TMP_Text feedbackText;
|
||||
|
||||
/// <summary>
|
||||
/// Reference to the progress bar
|
||||
/// </summary>
|
||||
public Slider feedbackProgress;
|
||||
|
||||
/// <summary>
|
||||
/// Reference to the progress bar image, so we can add fancy colors
|
||||
/// </summary>
|
||||
public Image feedbackProgressImage;
|
||||
|
||||
/// <summary>
|
||||
/// Timer to keep track of how long a incorrect sign is performed
|
||||
/// </summary>
|
||||
protected DateTime timer;
|
||||
|
||||
/// <summary>
|
||||
/// Current predicted sign
|
||||
/// </summary>
|
||||
protected string predictedSign = null;
|
||||
|
||||
/// <summary>
|
||||
/// Previous incorrect sign, so we can keep track whether the user is wrong or the user is still changing signs
|
||||
/// </summary>
|
||||
protected string previousIncorrectSign = null;
|
||||
|
||||
/// <summary>
|
||||
/// Start is called before the first frame update
|
||||
/// </summary>
|
||||
void Start()
|
||||
{
|
||||
StartController();
|
||||
|
||||
signPredictor.SetModel(ModelIndex.FINGERSPELLING);
|
||||
AddSelfAsListener();
|
||||
}
|
||||
/// <summary>
|
||||
/// Called at the start of the scene AND when the scene is replayed
|
||||
/// </summary>
|
||||
public void StartController()
|
||||
{
|
||||
// Make sure the mode starts at zero
|
||||
mode = 0;
|
||||
@@ -240,19 +274,6 @@ public class HangmanController : MonoBehaviour
|
||||
user.minigames.Add(progress);
|
||||
}
|
||||
userList.Save();
|
||||
|
||||
// Hangman always uses fingerspelling
|
||||
feedback.signPredictor.ChangeModel(ModelIndex.FINGERSPELLING);
|
||||
|
||||
// Set calllbacks
|
||||
feedback.getSignCallback = () =>
|
||||
{
|
||||
return "A";
|
||||
};
|
||||
feedback.predictSignCallback = (sign) =>
|
||||
{
|
||||
currentsign = sign;
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -281,7 +302,7 @@ public class HangmanController : MonoBehaviour
|
||||
DeleteWord();
|
||||
DisplayWord(currentWord);
|
||||
|
||||
replayButton.onClick.AddListener(Start);
|
||||
replayButton.onClick.AddListener(StartController);
|
||||
// Call to display the first image, corresponding to a clean image.
|
||||
ChangeSprite();
|
||||
}
|
||||
@@ -352,7 +373,7 @@ public class HangmanController : MonoBehaviour
|
||||
{
|
||||
if (mode == 1)
|
||||
{
|
||||
if (currentsign != "")
|
||||
if (currentsign != null && currentsign != "")
|
||||
{
|
||||
char letter = currentsign.ToLower()[0];
|
||||
currentsign = "";
|
||||
@@ -393,7 +414,7 @@ public class HangmanController : MonoBehaviour
|
||||
// For the first input char given by the user, check if the letter is in the word that needs to be spelled.
|
||||
|
||||
// Check to make sure the inputfield is not empty
|
||||
if (currentsign != "")
|
||||
if (currentsign != null && currentsign != "")
|
||||
{
|
||||
char firstLetter = currentsign.ToLower()[0];
|
||||
currentsign = "";
|
||||
@@ -543,21 +564,6 @@ public class HangmanController : MonoBehaviour
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Randomly shuffle the list of words
|
||||
/// </summary>
|
||||
private void ShuffleWords()
|
||||
{
|
||||
for (int i = words.Length - 1; i > 0; i--)
|
||||
{
|
||||
// Generate a random index between 0 and i (inclusive)
|
||||
int j = UnityEngine.Random.Range(0, i + 1);
|
||||
|
||||
// Swap the values at indices i and j
|
||||
(words[j], words[i]) = (words[i], words[j]);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Update and save the scores
|
||||
/// </summary>
|
||||
@@ -731,4 +737,97 @@ public class HangmanController : MonoBehaviour
|
||||
rank++;
|
||||
}
|
||||
}
|
||||
/// <summary>
|
||||
/// The updateFunction that is called when new probabilities become available
|
||||
/// </summary>
|
||||
/// <returns></returns>
|
||||
protected override IEnumerator UpdateFeedback()
|
||||
{
|
||||
// Get current sign
|
||||
string currentSign = "A";
|
||||
// Get the predicted sign
|
||||
if (signPredictor != null && signPredictor.learnableProbabilities != null &&
|
||||
currentSign != null && signPredictor.learnableProbabilities.ContainsKey(currentSign))
|
||||
{
|
||||
float accuracy = signPredictor.learnableProbabilities[currentSign];
|
||||
if (feedbackText != null && feedbackProgressImage != null)
|
||||
{
|
||||
if (accuracy > 0.90)
|
||||
{
|
||||
feedbackText.text = "Goed";
|
||||
feedbackText.color = Color.green;
|
||||
feedbackProgressImage.color = Color.green;
|
||||
}
|
||||
else if (accuracy > 0.80)
|
||||
{
|
||||
feedbackText.text = "Bijna...";
|
||||
Color col = new Color(0xff / 255.0f, 0x66 / 255.0f, 0x00 / 255.0f);
|
||||
feedbackText.color = col;
|
||||
feedbackProgressImage.color = col;
|
||||
}
|
||||
else
|
||||
{
|
||||
feedbackText.text = "Detecteren...";
|
||||
feedbackText.color = Color.red;
|
||||
feedbackProgressImage.color = Color.red;
|
||||
}
|
||||
|
||||
float oldValue = feedbackProgress.value;
|
||||
// use an exponential scale
|
||||
float newValue = Mathf.Exp(4 * (accuracy - 1.0f));
|
||||
feedbackProgress.gameObject.Tween("FeedbackUpdate", oldValue, newValue, 0.2f, TweenScaleFunctions.CubicEaseInOut, (t) =>
|
||||
{
|
||||
if (feedbackProgress != null)
|
||||
{
|
||||
feedbackProgress.value = t.CurrentValue;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Check whether (in)correct sign has high accuracy
|
||||
foreach (var kv in signPredictor.learnableProbabilities)
|
||||
{
|
||||
if (kv.Value > 0.90)
|
||||
{
|
||||
predictedSign = kv.Key;
|
||||
// Correct sign
|
||||
if (predictedSign == currentSign)
|
||||
{
|
||||
yield return new WaitForSeconds(1.0f);
|
||||
CheckEquality(predictedSign);
|
||||
timer = DateTime.Now;
|
||||
predictedSign = null;
|
||||
previousIncorrectSign = null;
|
||||
}
|
||||
// Incorrect sign
|
||||
else
|
||||
{
|
||||
if (previousIncorrectSign != predictedSign)
|
||||
{
|
||||
timer = DateTime.Now;
|
||||
previousIncorrectSign = predictedSign;
|
||||
}
|
||||
else if (DateTime.Now - timer > TimeSpan.FromSeconds(2.0f))
|
||||
{
|
||||
CheckEquality(predictedSign);
|
||||
timer = DateTime.Now;
|
||||
predictedSign = null;
|
||||
previousIncorrectSign = null;
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
else if (feedbackProgress != null)
|
||||
{
|
||||
|
||||
feedbackProgress.value = 0.0f;
|
||||
}
|
||||
|
||||
}
|
||||
private void CheckEquality(string sign)
|
||||
{
|
||||
currentsign = sign;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,95 +0,0 @@
|
||||
using UnityEngine.UI;
|
||||
using UnityEngine;
|
||||
|
||||
public class WebCam : MonoBehaviour
|
||||
{
|
||||
/// <summary>
|
||||
/// Index of the current camera
|
||||
/// </summary>
|
||||
protected int camdex = 0;
|
||||
|
||||
/// <summary>
|
||||
/// Texture to paste on the display
|
||||
/// </summary>
|
||||
protected WebCamTexture tex;
|
||||
|
||||
/// <summary>
|
||||
/// Display for the video feed
|
||||
/// </summary>
|
||||
public RawImage display;
|
||||
|
||||
/// <summary>
|
||||
/// Setup the webcam correctly
|
||||
/// </summary>
|
||||
void Awake()
|
||||
{
|
||||
WebCamDevice device = WebCamTexture.devices[camdex];
|
||||
tex = new WebCamTexture(device.name);
|
||||
display.texture = tex;
|
||||
|
||||
tex.Play();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Function to toggle between stopping and starting
|
||||
/// </summary>
|
||||
/*
|
||||
public void toggle()
|
||||
{
|
||||
if (tex.isPlaying)
|
||||
{
|
||||
tex.Stop();
|
||||
}
|
||||
else
|
||||
{
|
||||
tex.Play();
|
||||
}
|
||||
}
|
||||
*/
|
||||
public void PlayCam()
|
||||
{
|
||||
if (!tex.isPlaying) tex.Play();
|
||||
}
|
||||
|
||||
public void StopCam()
|
||||
{
|
||||
if (tex.isPlaying) tex.Stop();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Swap webcam by cycling through the `WebCamTexture.devices` list
|
||||
/// </summary>
|
||||
public void SwapCam()
|
||||
{
|
||||
if (WebCamTexture.devices.Length > 0)
|
||||
{
|
||||
// Stop the old camera
|
||||
display.texture = null;
|
||||
tex.Stop();
|
||||
tex = null;
|
||||
|
||||
// Find the new camera
|
||||
camdex += 1;
|
||||
camdex %= WebCamTexture.devices.Length;
|
||||
|
||||
// Start the new camera
|
||||
WebCamDevice device = WebCamTexture.devices[camdex];
|
||||
tex = new WebCamTexture(device.name);
|
||||
display.texture = tex;
|
||||
|
||||
tex.Play();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Scene changing is implemented here to avoid problems with webcam
|
||||
/// </summary>
|
||||
public void GotoThemeSelection()
|
||||
{
|
||||
display.texture = null;
|
||||
tex.Stop();
|
||||
tex = null;
|
||||
|
||||
SystemController.GetInstance().BackToPreviousScene();
|
||||
}
|
||||
}
|
||||
@@ -1,11 +0,0 @@
|
||||
fileFormatVersion: 2
|
||||
guid: 5db51e2552e03de4b9e7e91b5746adbc
|
||||
MonoImporter:
|
||||
externalObjects: {}
|
||||
serializedVersion: 2
|
||||
defaultReferences: []
|
||||
executionOrder: 0
|
||||
icon: {instanceID: 0}
|
||||
userData:
|
||||
assetBundleName:
|
||||
assetBundleVariant:
|
||||
8
Assets/MediaPipeUnity/Interfaces.meta
Normal file
8
Assets/MediaPipeUnity/Interfaces.meta
Normal file
@@ -0,0 +1,8 @@
|
||||
fileFormatVersion: 2
|
||||
guid: 73c615986873dc246893879daf74c05d
|
||||
folderAsset: yes
|
||||
DefaultImporter:
|
||||
externalObjects: {}
|
||||
userData:
|
||||
assetBundleName:
|
||||
assetBundleVariant:
|
||||
14
Assets/MediaPipeUnity/Interfaces/Listener.cs
Normal file
14
Assets/MediaPipeUnity/Interfaces/Listener.cs
Normal file
@@ -0,0 +1,14 @@
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
using UnityEngine;
|
||||
/// <summary>
|
||||
/// Listener interface with an IEnumerator as its processing-function.
|
||||
/// </summary>
|
||||
public interface Listener
|
||||
{
|
||||
/// <summary>
|
||||
/// The function that is called by the publisher.
|
||||
/// </summary>
|
||||
/// <returns></returns>
|
||||
public IEnumerator ProcessIncomingCall();
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
fileFormatVersion: 2
|
||||
guid: 44e682a32ee15cc489bf50f3a06f717b
|
||||
guid: e4c1da9896d9ba2449549a016b5fd15e
|
||||
MonoImporter:
|
||||
externalObjects: {}
|
||||
serializedVersion: 2
|
||||
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"name": "SignPredictorInterfaces",
|
||||
"rootNamespace": "",
|
||||
"references": [],
|
||||
"includePlatforms": [],
|
||||
"excludePlatforms": [],
|
||||
"allowUnsafeCode": false,
|
||||
"overrideReferences": false,
|
||||
"precompiledReferences": [],
|
||||
"autoReferenced": true,
|
||||
"defineConstraints": [],
|
||||
"versionDefines": [],
|
||||
"noEngineReferences": false
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
fileFormatVersion: 2
|
||||
guid: f55a02e98b01bc849b30d9650ccd8f15
|
||||
AssemblyDefinitionImporter:
|
||||
externalObjects: {}
|
||||
userData:
|
||||
assetBundleName:
|
||||
assetBundleVariant:
|
||||
@@ -9,7 +9,6 @@ GameObject:
|
||||
serializedVersion: 6
|
||||
m_Component:
|
||||
- component: {fileID: 4318122119930585316}
|
||||
- component: {fileID: 4318122119930585317}
|
||||
m_Layer: 5
|
||||
m_Name: Feedback
|
||||
m_TagString: Untagged
|
||||
@@ -39,22 +38,6 @@ RectTransform:
|
||||
m_AnchoredPosition: {x: 0, y: 200}
|
||||
m_SizeDelta: {x: 500, y: 150}
|
||||
m_Pivot: {x: 0.5, y: 0}
|
||||
--- !u!114 &4318122119930585317
|
||||
MonoBehaviour:
|
||||
m_ObjectHideFlags: 0
|
||||
m_CorrespondingSourceObject: {fileID: 0}
|
||||
m_PrefabInstance: {fileID: 0}
|
||||
m_PrefabAsset: {fileID: 0}
|
||||
m_GameObject: {fileID: 4318122119930585319}
|
||||
m_Enabled: 1
|
||||
m_EditorHideFlags: 0
|
||||
m_Script: {fileID: 11500000, guid: 44e682a32ee15cc489bf50f3a06f717b, type: 3}
|
||||
m_Name:
|
||||
m_EditorClassIdentifier:
|
||||
feedbackText: {fileID: 4318122120222767928}
|
||||
feedbackProgress: {fileID: 4318122119968934242}
|
||||
feedbackProgressImage: {fileID: 4318122120334233319}
|
||||
signPredictor: {fileID: 0}
|
||||
--- !u!1 &4318122119968934244
|
||||
GameObject:
|
||||
m_ObjectHideFlags: 0
|
||||
|
||||
@@ -17,4 +17,4 @@ MonoBehaviour:
|
||||
- index: 0
|
||||
model: {fileID: 5022602860645237092, guid: e6d85df707405ad4f97c23b07227ee99, type: 3}
|
||||
- index: 1
|
||||
model: {fileID: 5022602860645237092, guid: e6d85df707405ad4f97c23b07227ee99, type: 3}
|
||||
model: {fileID: 0}
|
||||
|
||||
40
Assets/MediaPipeUnity/Scripts/AbstractFeedback.cs
Normal file
40
Assets/MediaPipeUnity/Scripts/AbstractFeedback.cs
Normal file
@@ -0,0 +1,40 @@
|
||||
using DigitalRuby.Tween;
|
||||
using Mediapipe.Unity.Tutorial;
|
||||
using System;
|
||||
using System.Collections;
|
||||
using TMPro;
|
||||
using UnityEngine;
|
||||
using UnityEngine.Events;
|
||||
using UnityEngine.UI;
|
||||
|
||||
/// <summary>
|
||||
/// Class to display feedback during a course
|
||||
/// </summary>
|
||||
public abstract class AbstractFeedback : MonoBehaviour, Listener
|
||||
{
|
||||
|
||||
/// <summary>
|
||||
/// Reference to the sign predictor
|
||||
/// </summary>
|
||||
public SignPredictor signPredictor;
|
||||
/// <summary>
|
||||
/// The function that is called by the publisher on all its listeners
|
||||
/// </summary>
|
||||
/// <returns></returns>
|
||||
public IEnumerator ProcessIncomingCall()
|
||||
{
|
||||
yield return StartCoroutine(UpdateFeedback());
|
||||
}
|
||||
/// <summary>
|
||||
/// A function to add yourself as listener to the signPredictor you are holding
|
||||
/// </summary>
|
||||
public void AddSelfAsListener()
|
||||
{
|
||||
signPredictor.listeners.Add(this);
|
||||
}
|
||||
/// <summary>
|
||||
/// The function that holds the logic to process the new probabilities of the signPredictor
|
||||
/// </summary>
|
||||
/// <returns></returns>
|
||||
protected abstract IEnumerator UpdateFeedback();
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
fileFormatVersion: 2
|
||||
guid: a62d2b8bded916443835dc19010b83c1
|
||||
guid: 7b5ac794337a54143a6e3077483d96c9
|
||||
MonoImporter:
|
||||
externalObjects: {}
|
||||
serializedVersion: 2
|
||||
@@ -1,182 +0,0 @@
|
||||
using DigitalRuby.Tween;
|
||||
using Mediapipe.Unity.Tutorial;
|
||||
using System;
|
||||
using System.Collections;
|
||||
using TMPro;
|
||||
using UnityEngine;
|
||||
using UnityEngine.Events;
|
||||
using UnityEngine.UI;
|
||||
|
||||
/// <summary>
|
||||
/// Class to display feedback during a course
|
||||
/// </summary>
|
||||
public class Feedback : MonoBehaviour
|
||||
{
|
||||
/// <summary>
|
||||
/// Reference to the feedback field
|
||||
/// </summary>
|
||||
public TMP_Text feedbackText;
|
||||
|
||||
/// <summary>
|
||||
/// Reference to the progress bar
|
||||
/// </summary>
|
||||
public Slider feedbackProgress;
|
||||
|
||||
/// <summary>
|
||||
/// Reference to the progress bar image, so we can add fancy colors
|
||||
/// </summary>
|
||||
public Image feedbackProgressImage;
|
||||
|
||||
/// <summary>
|
||||
/// Reference to the sign predictor
|
||||
/// </summary>
|
||||
public SignPredictor signPredictor;
|
||||
|
||||
/// <summary>
|
||||
/// Callback for getting the correct sign
|
||||
/// </summary>
|
||||
public Func<string> getSignCallback;
|
||||
|
||||
/// <summary>
|
||||
/// Callback to initiate the next sign
|
||||
/// </summary>
|
||||
public UnityAction<string> predictSignCallback;
|
||||
|
||||
/// <summary>
|
||||
/// Timer to keep track of how long a incorrect sign is performed
|
||||
/// </summary>
|
||||
private DateTime timer;
|
||||
|
||||
/// <summary>
|
||||
/// Current predicted sign
|
||||
/// </summary>
|
||||
private string predictedSign = null;
|
||||
|
||||
/// <summary>
|
||||
/// Previous incorrect sign, so we can keep track whether the user is wrong or the user is still changing signs
|
||||
/// </summary>
|
||||
private string previousIncorrectSign = null;
|
||||
|
||||
/// <summary>
|
||||
/// Start is called before the first frame update
|
||||
/// </summary>
|
||||
void Start()
|
||||
{
|
||||
// Start the coroutine to update the scale every 200 milliseconds
|
||||
StartCoroutine(UpdateFeedback());
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// UpdateScale updates the progress bar every 200ms, updated the feedback text, and progress bar color
|
||||
/// If a high enough accuracy is detected, it will go to the next sign
|
||||
/// </summary>
|
||||
/// <returns></returns>
|
||||
IEnumerator UpdateFeedback()
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
if (getSignCallback != null && predictSignCallback != null)
|
||||
{
|
||||
|
||||
// Get current sign
|
||||
string currentSign = getSignCallback();
|
||||
// Get the predicted sign
|
||||
if (signPredictor != null && signPredictor.learnableProbabilities != null &&
|
||||
currentSign != null && signPredictor.learnableProbabilities.ContainsKey(currentSign))
|
||||
{
|
||||
float accuracy = signPredictor.learnableProbabilities[currentSign];
|
||||
if (feedbackText != null && feedbackProgressImage != null){
|
||||
if (accuracy > 0.98)
|
||||
{
|
||||
// TODO: fix emojis
|
||||
feedbackText.text = "✨ Perfect ✨";
|
||||
Color col = new Color(0xff / 255.0f, 0xcc / 255.0f, 0x00 / 255.0f);
|
||||
feedbackText.color = col;
|
||||
feedbackProgressImage.color = col;
|
||||
}
|
||||
else if (accuracy > 0.95)
|
||||
{
|
||||
feedbackText.text = "Super!";
|
||||
Color col = new Color(0x00 / 255.0f, 0xff / 255.0f, 0xcc / 255.0f);
|
||||
feedbackText.color = col;
|
||||
feedbackProgressImage.color = col;
|
||||
}
|
||||
else if (accuracy > 0.90)
|
||||
{
|
||||
feedbackText.text = "Goed";
|
||||
feedbackText.color = Color.green;
|
||||
feedbackProgressImage.color = Color.green;
|
||||
}
|
||||
else if (accuracy > 0.80)
|
||||
{
|
||||
feedbackText.text = "Bijna...";
|
||||
Color col = new Color(0xff / 255.0f, 0x66 / 255.0f, 0x00 / 255.0f);
|
||||
feedbackText.color = col;
|
||||
feedbackProgressImage.color = col;
|
||||
}
|
||||
else
|
||||
{
|
||||
feedbackText.text = "Detecteren...";
|
||||
feedbackText.color = Color.red;
|
||||
feedbackProgressImage.color = Color.red;
|
||||
}
|
||||
|
||||
float oldValue = feedbackProgress.value;
|
||||
// use an exponential scale
|
||||
float newValue = Mathf.Exp(4 * (accuracy - 1.0f));
|
||||
feedbackProgress.gameObject.Tween("FeedbackUpdate", oldValue, newValue, 0.2f, TweenScaleFunctions.CubicEaseInOut, (t) =>
|
||||
{
|
||||
if (feedbackProgress != null)
|
||||
{
|
||||
feedbackProgress.value = t.CurrentValue;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Check whether (in)correct sign has high accuracy
|
||||
foreach (var kv in signPredictor.learnableProbabilities)
|
||||
{
|
||||
if (kv.Value > 0.90)
|
||||
{
|
||||
predictedSign = kv.Key;
|
||||
// Correct sign
|
||||
if (predictedSign == currentSign)
|
||||
{
|
||||
yield return new WaitForSeconds(1.0f);
|
||||
predictSignCallback(predictedSign);
|
||||
timer = DateTime.Now;
|
||||
predictedSign = null;
|
||||
previousIncorrectSign = null;
|
||||
}
|
||||
// Incorrect sign
|
||||
else
|
||||
{
|
||||
if (previousIncorrectSign != predictedSign)
|
||||
{
|
||||
timer = DateTime.Now;
|
||||
previousIncorrectSign = predictedSign;
|
||||
}
|
||||
else if (DateTime.Now - timer > TimeSpan.FromSeconds(2.0f))
|
||||
{
|
||||
predictSignCallback(predictedSign);
|
||||
timer = DateTime.Now;
|
||||
predictedSign = null;
|
||||
previousIncorrectSign = null;
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
else if(feedbackProgress != null)
|
||||
{
|
||||
|
||||
feedbackProgress.value = 0.0f;
|
||||
}
|
||||
}
|
||||
|
||||
// Wait for 200 milliseconds before updating the scale again
|
||||
yield return new WaitForSeconds(0.2f);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -7,7 +7,8 @@
|
||||
"GUID:04c4d86a70aa56c55a78c61f1ab1a56d",
|
||||
"GUID:edc93f477bb73a743a97d6882ed330b3",
|
||||
"GUID:58e104b97fb3752438ada2902a36dcbf",
|
||||
"GUID:7f2d0ee6dd21e1d4eb25b71b7a749d25"
|
||||
"GUID:7f2d0ee6dd21e1d4eb25b71b7a749d25",
|
||||
"GUID:f55a02e98b01bc849b30d9650ccd8f15"
|
||||
],
|
||||
"includePlatforms": [],
|
||||
"excludePlatforms": [],
|
||||
|
||||
@@ -141,6 +141,8 @@ namespace Mediapipe.Unity.Tutorial
|
||||
/// </summary>
|
||||
private Tensor inputTensor;
|
||||
|
||||
public List<Listener> listeners = new List<Listener>();
|
||||
|
||||
/// <summary>
|
||||
/// Google Mediapipe setup & run
|
||||
/// </summary>
|
||||
@@ -159,6 +161,7 @@ namespace Mediapipe.Unity.Tutorial
|
||||
|
||||
webcamTexture.Play();
|
||||
|
||||
|
||||
yield return new WaitUntil(() => webcamTexture.width > 16);
|
||||
|
||||
// Set webcam aspect ratio
|
||||
@@ -172,6 +175,8 @@ namespace Mediapipe.Unity.Tutorial
|
||||
screen2.rectTransform.sizeDelta = new Vector2(screen2.rectTransform.sizeDelta.y * webcamAspect, (screen2.rectTransform.sizeDelta.y));
|
||||
}
|
||||
|
||||
if (modelList.GetCurrentModel() != null)
|
||||
{
|
||||
// TODO this method is kinda meh you should use
|
||||
inputTexture = new Texture2D(width, height, TextureFormat.RGBA32, false);
|
||||
pixelData = new Color32[width * height];
|
||||
@@ -207,7 +212,6 @@ namespace Mediapipe.Unity.Tutorial
|
||||
|
||||
|
||||
keypointManager = new KeypointManager(modelInfoFile);
|
||||
|
||||
// check if model exists at path
|
||||
//var model = ModelLoader.Load(Resources.Load<NNModel>("Models/Fingerspelling/model_A-L"));
|
||||
worker = modelList.GetCurrentModel().CreateWorker();
|
||||
@@ -215,15 +219,14 @@ namespace Mediapipe.Unity.Tutorial
|
||||
StartCoroutine(SignRecognitionCoroutine());
|
||||
StartCoroutine(MediapipeCoroutine());
|
||||
}
|
||||
|
||||
public void ChangeModel(ModelIndex index)
|
||||
}
|
||||
/// <summary>
|
||||
/// Called at the start of course/Minigame, will set the model before the start of SIgnPredictor is called.
|
||||
/// </summary>
|
||||
/// <param name="index">The index of the model to be used</param>
|
||||
public void SetModel(ModelIndex index)
|
||||
{
|
||||
this.modelList.SetCurrentModel(index);
|
||||
// If a worker already existed, we throw it out
|
||||
worker?.Dispose();
|
||||
|
||||
// Add a new worker for the new model
|
||||
worker = modelList.GetCurrentModel().CreateWorker();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -325,6 +328,10 @@ namespace Mediapipe.Unity.Tutorial
|
||||
learnableProbabilities.Add(((char)(i + 65)).ToString(), softmaxedOutput2[i]);
|
||||
}
|
||||
//Debug.Log($"prob = [{learnableProbabilities.Aggregate(" ", (t, kv) => $"{t}{kv.Key}:{kv.Value} ")}]");
|
||||
foreach(Listener listener in listeners)
|
||||
{
|
||||
yield return listener.ProcessIncomingCall();
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
|
||||
@@ -7,7 +7,8 @@
|
||||
"InterfacesScripts",
|
||||
"Unity.TextMeshPro",
|
||||
"SpellingBeeScripts",
|
||||
"AccountsScripts"
|
||||
"AccountsScripts",
|
||||
"SignPredictor"
|
||||
],
|
||||
"includePlatforms": [],
|
||||
"excludePlatforms": [],
|
||||
|
||||
@@ -1741,13 +1741,35 @@ RectTransform:
|
||||
m_PrefabAsset: {fileID: 0}
|
||||
--- !u!114 &967164045 stripped
|
||||
MonoBehaviour:
|
||||
m_CorrespondingSourceObject: {fileID: 4318122119930585317, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3}
|
||||
m_CorrespondingSourceObject: {fileID: 4318122120334233319, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3}
|
||||
m_PrefabInstance: {fileID: 967164043}
|
||||
m_PrefabAsset: {fileID: 0}
|
||||
m_GameObject: {fileID: 0}
|
||||
m_Enabled: 1
|
||||
m_EditorHideFlags: 0
|
||||
m_Script: {fileID: 11500000, guid: 44e682a32ee15cc489bf50f3a06f717b, type: 3}
|
||||
m_Script: {fileID: 11500000, guid: fe87c0e1cc204ed48ad3b37840f39efc, type: 3}
|
||||
m_Name:
|
||||
m_EditorClassIdentifier:
|
||||
--- !u!114 &967164046 stripped
|
||||
MonoBehaviour:
|
||||
m_CorrespondingSourceObject: {fileID: 4318122119968934242, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3}
|
||||
m_PrefabInstance: {fileID: 967164043}
|
||||
m_PrefabAsset: {fileID: 0}
|
||||
m_GameObject: {fileID: 0}
|
||||
m_Enabled: 1
|
||||
m_EditorHideFlags: 0
|
||||
m_Script: {fileID: 11500000, guid: 67db9e8f0e2ae9c40bc1e2b64352a6b4, type: 3}
|
||||
m_Name:
|
||||
m_EditorClassIdentifier:
|
||||
--- !u!114 &967164047 stripped
|
||||
MonoBehaviour:
|
||||
m_CorrespondingSourceObject: {fileID: 4318122120222767928, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3}
|
||||
m_PrefabInstance: {fileID: 967164043}
|
||||
m_PrefabAsset: {fileID: 0}
|
||||
m_GameObject: {fileID: 0}
|
||||
m_Enabled: 1
|
||||
m_EditorHideFlags: 0
|
||||
m_Script: {fileID: 11500000, guid: f4688fdb7df04437aeb418b961361dc5, type: 3}
|
||||
m_Name:
|
||||
m_EditorClassIdentifier:
|
||||
--- !u!1 &978093274
|
||||
@@ -3584,6 +3606,7 @@ MonoBehaviour:
|
||||
m_Script: {fileID: 11500000, guid: 44fbed5ae228de39b9f727def7578d06, type: 3}
|
||||
m_Name:
|
||||
m_EditorClassIdentifier:
|
||||
signPredictor: {fileID: 1592592444}
|
||||
themeList: {fileID: 11400000, guid: a247e2ce790f0f746a3bc521e6ab7d58, type: 2}
|
||||
userList: {fileID: 11400000, guid: 072bec636a40f7e4e93b0ac624a3bda2, type: 2}
|
||||
minigame: {fileID: 11400000, guid: 8a087d241d652634eb4f6352267ea7dc, type: 2}
|
||||
@@ -3594,8 +3617,10 @@ MonoBehaviour:
|
||||
timerText: {fileID: 1843239267}
|
||||
bonusTimeText: {fileID: 1812475780}
|
||||
Scoreboard: {fileID: 862382568}
|
||||
feedback: {fileID: 967164045}
|
||||
gameEndedPanel: {fileID: 757133117}
|
||||
feedbackText: {fileID: 967164047}
|
||||
feedbackProgress: {fileID: 967164046}
|
||||
feedbackProgressImage: {fileID: 967164045}
|
||||
--- !u!1 &1499197558
|
||||
GameObject:
|
||||
m_ObjectHideFlags: 0
|
||||
@@ -3957,7 +3982,6 @@ MonoBehaviour:
|
||||
m_Name:
|
||||
m_EditorClassIdentifier:
|
||||
modelList: {fileID: 11400000, guid: 39516e4e6e56f0f4f80647d9c4d8034c, type: 2}
|
||||
model: {fileID: 5022602860645237092, guid: e6d85df707405ad4f97c23b07227ee99, type: 3}
|
||||
modelInfoFile: {fileID: 4900000, guid: fb8b51022bdcd654a9f29c054832a1b5, type: 3}
|
||||
configAsset: {fileID: 4900000, guid: 6288c43cdca97374782dac1ea87aa029, type: 3}
|
||||
screen: {fileID: 1743003084}
|
||||
|
||||
@@ -5,8 +5,9 @@ using System.Linq;
|
||||
using TMPro;
|
||||
using UnityEngine;
|
||||
using UnityEngine.UI;
|
||||
using DigitalRuby.Tween;
|
||||
|
||||
public partial class SpellingBeeController : MonoBehaviour
|
||||
public partial class SpellingBeeController : AbstractFeedback
|
||||
{
|
||||
/// <summary>
|
||||
/// All of the words that can be used in this session
|
||||
@@ -136,20 +137,55 @@ public partial class SpellingBeeController : MonoBehaviour
|
||||
/// </summary>
|
||||
public Transform Scoreboard;
|
||||
|
||||
/// <summary>
|
||||
/// Accuracy feeback object
|
||||
/// </summary>
|
||||
public Feedback feedback;
|
||||
|
||||
/// <summary>
|
||||
/// Reference to the gameEnded panel, so we can update its display
|
||||
/// </summary>
|
||||
public GameObject gameEndedPanel;
|
||||
|
||||
/// <summary>
|
||||
/// Reference to the feedback field
|
||||
/// </summary>
|
||||
public TMP_Text feedbackText;
|
||||
|
||||
/// <summary>
|
||||
/// Reference to the progress bar
|
||||
/// </summary>
|
||||
public Slider feedbackProgress;
|
||||
|
||||
/// <summary>
|
||||
/// Reference to the progress bar image, so we can add fancy colors
|
||||
/// </summary>
|
||||
public Image feedbackProgressImage;
|
||||
|
||||
/// <summary>
|
||||
/// Timer to keep track of how long a incorrect sign is performed
|
||||
/// </summary>
|
||||
protected DateTime timer;
|
||||
|
||||
/// <summary>
|
||||
/// Current predicted sign
|
||||
/// </summary>
|
||||
protected string predictedSign = null;
|
||||
|
||||
/// <summary>
|
||||
/// Previous incorrect sign, so we can keep track whether the user is wrong or the user is still changing signs
|
||||
/// </summary>
|
||||
protected string previousIncorrectSign = null;
|
||||
|
||||
/// <summary>
|
||||
/// Start is called before the first frame update
|
||||
/// </summary>
|
||||
public void Start()
|
||||
{
|
||||
StartController();
|
||||
|
||||
signPredictor.SetModel(currentTheme.modelIndex);
|
||||
AddSelfAsListener();
|
||||
}
|
||||
/// <summary>
|
||||
/// Is called at the start of the scene AND when the game is replayed
|
||||
/// </summary>
|
||||
public void StartController()
|
||||
{
|
||||
correctLetters = 0;
|
||||
incorrectLetters = 0;
|
||||
@@ -182,29 +218,10 @@ public partial class SpellingBeeController : MonoBehaviour
|
||||
userList.Save();
|
||||
|
||||
currentTheme = minigame.themeList.themes[minigame.themeList.currentThemeIndex];
|
||||
feedback.signPredictor.ChangeModel(currentTheme.modelIndex);
|
||||
//feedback.signPredictor.ChangeModel(currentTheme.modelIndex);
|
||||
words.AddRange(currentTheme.learnables);
|
||||
ShuffleWords();
|
||||
NextWord();
|
||||
|
||||
// Set calllbacks
|
||||
feedback.getSignCallback = () =>
|
||||
{
|
||||
if (letterIndex < currentWord.Length)
|
||||
{
|
||||
return currentWord[letterIndex].ToString().ToUpper();
|
||||
}
|
||||
return null;
|
||||
};
|
||||
feedback.predictSignCallback = (sign) =>
|
||||
{
|
||||
bool successful = sign.ToUpper() == currentWord[letterIndex].ToString().ToUpper();
|
||||
if (successful)
|
||||
{
|
||||
AddSeconds(secondsPerLetter);
|
||||
}
|
||||
NextLetter(successful);
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -453,4 +470,116 @@ public partial class SpellingBeeController : MonoBehaviour
|
||||
{
|
||||
yield return new WaitForSecondsRealtime(2);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The updateFunction that is called when new probabilities become available
|
||||
/// </summary>
|
||||
/// <returns></returns>
|
||||
protected override IEnumerator UpdateFeedback()
|
||||
{
|
||||
// Get current sign
|
||||
string currentSign = GetSign();
|
||||
// Get the predicted sign
|
||||
if (signPredictor != null && signPredictor.learnableProbabilities != null &&
|
||||
currentSign != null && signPredictor.learnableProbabilities.ContainsKey(currentSign))
|
||||
{
|
||||
float accuracy = signPredictor.learnableProbabilities[currentSign];
|
||||
if (feedbackText != null && feedbackProgressImage != null)
|
||||
{
|
||||
if (accuracy > 0.90)
|
||||
{
|
||||
feedbackText.text = "Goed";
|
||||
feedbackText.color = Color.green;
|
||||
feedbackProgressImage.color = Color.green;
|
||||
}
|
||||
else if (accuracy > 0.80)
|
||||
{
|
||||
feedbackText.text = "Bijna...";
|
||||
Color col = new Color(0xff / 255.0f, 0x66 / 255.0f, 0x00 / 255.0f);
|
||||
feedbackText.color = col;
|
||||
feedbackProgressImage.color = col;
|
||||
}
|
||||
else
|
||||
{
|
||||
feedbackText.text = "Detecteren...";
|
||||
feedbackText.color = Color.red;
|
||||
feedbackProgressImage.color = Color.red;
|
||||
}
|
||||
|
||||
float oldValue = feedbackProgress.value;
|
||||
// use an exponential scale
|
||||
float newValue = Mathf.Exp(4 * (accuracy - 1.0f));
|
||||
feedbackProgress.gameObject.Tween("FeedbackUpdate", oldValue, newValue, 0.2f, TweenScaleFunctions.CubicEaseInOut, (t) =>
|
||||
{
|
||||
if (feedbackProgress != null)
|
||||
{
|
||||
feedbackProgress.value = t.CurrentValue;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Check whether (in)correct sign has high accuracy
|
||||
foreach (var kv in signPredictor.learnableProbabilities)
|
||||
{
|
||||
if (kv.Value > 0.90)
|
||||
{
|
||||
predictedSign = kv.Key;
|
||||
// Correct sign
|
||||
if (predictedSign == currentSign)
|
||||
{
|
||||
yield return new WaitForSeconds(1.0f);
|
||||
predictSign(predictedSign);
|
||||
timer = DateTime.Now;
|
||||
predictedSign = null;
|
||||
previousIncorrectSign = null;
|
||||
}
|
||||
// Incorrect sign
|
||||
else
|
||||
{
|
||||
if (previousIncorrectSign != predictedSign)
|
||||
{
|
||||
timer = DateTime.Now;
|
||||
previousIncorrectSign = predictedSign;
|
||||
}
|
||||
else if (DateTime.Now - timer > TimeSpan.FromSeconds(2.0f))
|
||||
{
|
||||
predictSign(predictedSign);
|
||||
timer = DateTime.Now;
|
||||
predictedSign = null;
|
||||
previousIncorrectSign = null;
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
else if (feedbackProgress != null)
|
||||
{
|
||||
|
||||
feedbackProgress.value = 0.0f;
|
||||
}
|
||||
yield return null;
|
||||
}
|
||||
/// <summary>
|
||||
/// Function to get the current letter that needs to be signed
|
||||
/// </summary>
|
||||
/// <returns>the current letter that needs to be signed</returns>
|
||||
public string GetSign(){
|
||||
if (letterIndex<currentWord.Length){
|
||||
return currentWord[letterIndex].ToString().ToUpper();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
/// <summary>
|
||||
/// Function to confirm your prediction and check if it is correct.
|
||||
/// </summary>
|
||||
/// <param name="sign"></param>
|
||||
public void predictSign(string sign) {
|
||||
bool successful = sign.ToUpper() == currentWord[letterIndex].ToString().ToUpper();
|
||||
if (successful)
|
||||
{
|
||||
AddSeconds(secondsPerLetter);
|
||||
}
|
||||
NextLetter(successful);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,21 +3,7 @@ guid: 44fbed5ae228de39b9f727def7578d06
|
||||
MonoImporter:
|
||||
externalObjects: {}
|
||||
serializedVersion: 2
|
||||
defaultReferences:
|
||||
- input: {instanceID: 0}
|
||||
- endText: {instanceID: 0}
|
||||
- correctWordsText: {instanceID: 0}
|
||||
- correctLettersText: {instanceID: 0}
|
||||
- gameEndedPanel: {instanceID: 0}
|
||||
- replayButton: {instanceID: 0}
|
||||
- userList: {instanceID: 0}
|
||||
- minigame: {instanceID: 0}
|
||||
- letterPrefab: {instanceID: 0}
|
||||
- letterContainer: {instanceID: 0}
|
||||
- wordImage: {instanceID: 0}
|
||||
- timerText: {instanceID: 0}
|
||||
- Scoreboard: {instanceID: 0}
|
||||
- scoreboardEntry: {fileID: 9154151134820372555, guid: d4a3a228b08d61847acc6da35b44e52c, type: 3}
|
||||
defaultReferences: []
|
||||
executionOrder: 0
|
||||
icon: {instanceID: 0}
|
||||
userData:
|
||||
|
||||
@@ -6,8 +6,8 @@
|
||||
"GUID:1631ed2680c61245b8211d943c1639a8",
|
||||
"GUID:3444c67d5a3a93e5a95a48906078c372",
|
||||
"GUID:d0b6b39a21908f94fbbd9f2c196a9725",
|
||||
"GUID:5c2b5ba89f9e74e418232e154bc5cc7a",
|
||||
"GUID:7f2d0ee6dd21e1d4eb25b71b7a749d25"
|
||||
"GUID:7f2d0ee6dd21e1d4eb25b71b7a749d25",
|
||||
"GUID:58e104b97fb3752438ada2902a36dcbf"
|
||||
],
|
||||
"includePlatforms": [],
|
||||
"excludePlatforms": [],
|
||||
|
||||
Reference in New Issue
Block a user