WeSign/unity-application
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 6 Wiki Activity

Implemented changes to use abstract feedback

Browse Source
This commit is contained in:
CoudronJerome
2023-03-30 19:38:58 +02:00
parent a44532b94a
commit c9ebec1724
22 changed files with 559 additions and 332 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
Assets/Common/PlayModeTests/CommonPlayModeTests.asmdef
View File

@@ -8,7 +8,8 @@
"CommonScripts",
"InterfacesScripts",
"Unity.TextMeshPro",
"AccountsScripts"
"AccountsScripts",
"SignPredictor"
],
"includePlatforms": [],
"excludePlatforms": [],

3
Assets/Common/Scripts/CommonScripts.asmdef
View File

@@ -6,7 +6,8 @@
"GUID:63c63e721f65ebb7d871cb9ef49f4752",
"GUID:1631ed2680c61245b8211d943c1639a8",
"GUID:5c2b5ba89f9e74e418232e154bc5cc7a",
"GUID:7f2d0ee6dd21e1d4eb25b71b7a749d25"
"GUID:7f2d0ee6dd21e1d4eb25b71b7a749d25",
"GUID:d0b6b39a21908f94fbbd9f2c196a9725"
],
"includePlatforms": [],
"excludePlatforms": [],

69
Assets/Courses/Scenes/TemplateCourse.unity
View File

@@ -490,6 +490,10 @@ MonoBehaviour:
m_Script: {fileID: 11500000, guid: 6b3f784c065813a4a8364b1299284816, type: 3}
m_Name:
m_EditorClassIdentifier:
feedbackText: {fileID: 4318122121437849762}
feedbackProgress: {fileID: 4318122121437849761}
feedbackProgressImage: {fileID: 4318122121437849760}
signPredictor: {fileID: 883853268}
previewModel: {fileID: 5022602860645237092, guid: e6d85df707405ad4f97c23b07227ee99, type: 3}
feedbackProgressBar: {fileID: 4318122121437849759}
previewMessage: {fileID: 2070775951}
@@ -505,7 +509,6 @@ MonoBehaviour:
ResultsDecription: {fileID: 100123246}
CoursesButton: {fileID: 839294691}
timeSpent: {fileID: 77614869}
feedback: {fileID: 1714882683}
--- !u!1 &361280475
GameObject:
m_ObjectHideFlags: 0
@@ -1222,8 +1225,8 @@ MonoBehaviour:
m_OnClick:
m_PersistentCalls:
m_Calls:
- m_Target: {fileID: 301088551}
m_TargetAssemblyTypeName: TemplateCourse, Assembly-CSharp
- m_Target: {fileID: 1335886461}
m_TargetAssemblyTypeName: BackButton, CommonScripts
m_MethodName: Back
m_Mode: 1
m_Arguments:
@@ -1302,7 +1305,6 @@ MonoBehaviour:
m_Name:
m_EditorClassIdentifier:
modelList: {fileID: 11400000, guid: 39516e4e6e56f0f4f80647d9c4d8034c, type: 2}
model: {fileID: 5022602860645237092, guid: e6d85df707405ad4f97c23b07227ee99, type: 3}
modelInfoFile: {fileID: 4900000, guid: fb8b51022bdcd654a9f29c054832a1b5, type: 3}
configAsset: {fileID: 4900000, guid: 6288c43cdca97374782dac1ea87aa029, type: 3}
screen: {fileID: 378145456}
@@ -1840,6 +1842,17 @@ RectTransform:
m_CorrespondingSourceObject: {fileID: 8299246693487308515, guid: 3bccdf365a4fbea4d8fa1aa461d3dc5c, type: 3}
m_PrefabInstance: {fileID: 1335886459}
m_PrefabAsset: {fileID: 0}
--- !u!114 &1335886461 stripped
MonoBehaviour:
m_CorrespondingSourceObject: {fileID: 4518652150503380115, guid: 3bccdf365a4fbea4d8fa1aa461d3dc5c, type: 3}
m_PrefabInstance: {fileID: 1335886459}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 0}
m_Enabled: 1
m_EditorHideFlags: 0
m_Script: {fileID: 11500000, guid: c3dd279b546423e4a8a1b28819a6c4a1, type: 3}
m_Name:
m_EditorClassIdentifier:
--- !u!1 &1383144366
GameObject:
m_ObjectHideFlags: 0
@@ -2253,17 +2266,6 @@ RectTransform:
m_CorrespondingSourceObject: {fileID: 4318122119930585316, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3}
m_PrefabInstance: {fileID: 4318122121437849758}
m_PrefabAsset: {fileID: 0}
--- !u!114 &1714882683 stripped
MonoBehaviour:
m_CorrespondingSourceObject: {fileID: 4318122119930585317, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3}
m_PrefabInstance: {fileID: 4318122121437849758}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 4318122121437849759}
m_Enabled: 1
m_EditorHideFlags: 0
m_Script: {fileID: 11500000, guid: 44e682a32ee15cc489bf50f3a06f717b, type: 3}
m_Name:
m_EditorClassIdentifier:
--- !u!1 &1773033262
GameObject:
m_ObjectHideFlags: 0
@@ -2839,6 +2841,10 @@ PrefabInstance:
propertyPath: m_LocalEulerAnglesHint.z
value: 0
objectReference: {fileID: 0}
- target: {fileID: 4318122119930585317, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3}
propertyPath: m_Enabled
value: 0
objectReference: {fileID: 0}
- target: {fileID: 4318122119930585317, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3}
propertyPath: signPredictor
value:
@@ -2866,3 +2872,36 @@ GameObject:
m_CorrespondingSourceObject: {fileID: 4318122119930585319, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3}
m_PrefabInstance: {fileID: 4318122121437849758}
m_PrefabAsset: {fileID: 0}
--- !u!114 &4318122121437849760 stripped
MonoBehaviour:
m_CorrespondingSourceObject: {fileID: 4318122120334233319, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3}
m_PrefabInstance: {fileID: 4318122121437849758}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 0}
m_Enabled: 1
m_EditorHideFlags: 0
m_Script: {fileID: 11500000, guid: fe87c0e1cc204ed48ad3b37840f39efc, type: 3}
m_Name:
m_EditorClassIdentifier:
--- !u!114 &4318122121437849761 stripped
MonoBehaviour:
m_CorrespondingSourceObject: {fileID: 4318122119968934242, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3}
m_PrefabInstance: {fileID: 4318122121437849758}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 0}
m_Enabled: 1
m_EditorHideFlags: 0
m_Script: {fileID: 11500000, guid: 67db9e8f0e2ae9c40bc1e2b64352a6b4, type: 3}
m_Name:
m_EditorClassIdentifier:
--- !u!114 &4318122121437849762 stripped
MonoBehaviour:
m_CorrespondingSourceObject: {fileID: 4318122120222767928, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3}
m_PrefabInstance: {fileID: 4318122121437849758}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 0}
m_Enabled: 1
m_EditorHideFlags: 0
m_Script: {fileID: 11500000, guid: f4688fdb7df04437aeb418b961361dc5, type: 3}
m_Name:
m_EditorClassIdentifier:

4
Assets/Courses/Scripts/CourseScripts.asmdef
View File

@@ -6,7 +6,9 @@
"AccountsScripts",
"InterfacesScripts",
"SignPredictor",
"Unity.Barracuda"
"Unity.Barracuda",
"Tween",
"SignPredictorInterfaces"
],
"includePlatforms": [],
"excludePlatforms": [],

133
Assets/Courses/Scripts/TemplateCourse.cs
View File

@@ -1,14 +1,16 @@
using System;
using System.Collections;
using TMPro;
using Unity.Barracuda;
using UnityEngine;
using UnityEngine.UI;
using UnityEngine.Video;
using DigitalRuby.Tween;
/// <summary>
/// TemplateCourse scene manager
/// </summary>
public class TemplateCourse : MonoBehaviour
public class TemplateCourse : AbstractFeedback
{
// vvv TEMPORARY STUFF vvv
public NNModel previewModel;
@@ -112,11 +114,6 @@ public class TemplateCourse : MonoBehaviour
/// </summary>
public TMP_Text timeSpent;
/// <summary>
/// Reference to the feedback script on the Feedback prefab
/// </summary>
public Feedback feedback;
/// <summary>
/// This function is called when the script is initialised.
/// It inactivatis the popup, finds a webcam to use and links it via the WebcamTexture to the display RawImage.
@@ -124,18 +121,23 @@ public class TemplateCourse : MonoBehaviour
/// Then it checks whether or not the User has started the course yet, to possibly create a new progress atribute for the course.
/// Then it sets up the course-screen to display relevant information from the course-scriptable.
/// </summary>
void Awake()
void Start()
{
StartGameController();
signPredictor.SetModel(course.theme.modelIndex);
AddSelfAsListener();
}
public void StartGameController()
{
// Setting up course
course = courselist.courses[courselist.currentCourseIndex];
feedback.signPredictor.ChangeModel(course.theme.modelIndex);
maxWords = course.theme.learnables.Count;
// vvv TEMPORARY STUFF vvv
feedbackProgressBar.SetActive(course.theme.modelIndex != ModelIndex.NONE);
previewMessage.SetActive(course.theme.modelIndex == ModelIndex.NONE);
// Instead, the NONE-modelIndex points to Fingerspelling, which gives the same result
//feedback.signPredictor.model = previewModel;
// ^^^ TEMPORARY STUFF ^^^
// Create entry in current user for keeping track of progress
@@ -164,23 +166,6 @@ public class TemplateCourse : MonoBehaviour
ResultPanel.SetActive(false);
// Set the startTime
startMoment = DateTime.Now;
// Set callbacks
feedback.getSignCallback = () =>
{
if (currentWordIndex < course.theme.learnables.Count)
{
return course.theme.learnables[currentWordIndex].name;
}
return null;
};
feedback.predictSignCallback = (sign) =>
{
if (sign == course.theme.learnables[currentWordIndex].name)
{
NextSign();
}
};
}
/// <summary>
@@ -287,4 +272,98 @@ public class TemplateCourse : MonoBehaviour
progress.AddOrUpdate<float>("courseProgress", 1f);
userList.Save();
}
protected override IEnumerator UpdateFeedback()
{
// Get current sign
string currentSign = course.theme.learnables[currentWordIndex].name;
// Get the predicted sign
if (signPredictor != null && signPredictor.learnableProbabilities != null &&
currentSign != null && signPredictor.learnableProbabilities.ContainsKey(currentSign))
{
float accuracy = signPredictor.learnableProbabilities[currentSign];
if (feedbackText != null && feedbackProgressImage != null)
{
if (accuracy > 0.90)
{
feedbackText.text = "Goed";
feedbackText.color = Color.green;
feedbackProgressImage.color = Color.green;
}
else if (accuracy > 0.80)
{
feedbackText.text = "Bijna...";
Color col = new Color(0xff / 255.0f, 0x66 / 255.0f, 0x00 / 255.0f);
feedbackText.color = col;
feedbackProgressImage.color = col;
}
else
{
feedbackText.text = "Detecteren...";
feedbackText.color = Color.red;
feedbackProgressImage.color = Color.red;
}
float oldValue = feedbackProgress.value;
// use an exponential scale
float newValue = Mathf.Exp(4 * (accuracy - 1.0f));
feedbackProgress.gameObject.Tween("FeedbackUpdate", oldValue, newValue, 0.2f, TweenScaleFunctions.CubicEaseInOut, (t) =>
{
if (feedbackProgress != null)
{
feedbackProgress.value = t.CurrentValue;
}
});
}
// Check whether (in)correct sign has high accuracy
foreach (var kv in signPredictor.learnableProbabilities)
{
if (kv.Value > 0.90)
{
predictedSign = kv.Key;
// Correct sign
if (predictedSign == currentSign)
{
yield return new WaitForSeconds(1.0f);
CheckEquality(predictedSign);
timer = DateTime.Now;
predictedSign = null;
previousIncorrectSign = null;
}
// Incorrect sign
else
{
if (previousIncorrectSign != predictedSign)
{
timer = DateTime.Now;
previousIncorrectSign = predictedSign;
}
else if (DateTime.Now - timer > TimeSpan.FromSeconds(2.0f))
{
CheckEquality(predictedSign);
timer = DateTime.Now;
predictedSign = null;
previousIncorrectSign = null;
}
}
break;
}
}
}
else if (feedbackProgress != null)
{
feedbackProgress.value = 0.0f;
}
}
private void CheckEquality(string predicted)
{
if(predicted == course.theme.learnables[currentWordIndex].name)
{
NextSign();
}
}
}

16
Assets/Hangman/Scenes/Hangman.unity
View File

@@ -2570,7 +2570,7 @@ Transform:
m_ConstrainProportionsScale: 0
m_Children: []
m_Father: {fileID: 0}
m_RootOrder: 4
m_RootOrder: 5
m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
--- !u!1 &896343212
GameObject:
@@ -3507,7 +3507,7 @@ RectTransform:
- {fileID: 1892638588}
- {fileID: 56162990}
m_Father: {fileID: 0}
m_RootOrder: 2
m_RootOrder: 3
m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
m_AnchorMin: {x: 0, y: 0}
m_AnchorMax: {x: 0, y: 0}
@@ -3526,6 +3526,10 @@ MonoBehaviour:
m_Script: {fileID: 11500000, guid: 2db44635e0eb1e9429a2e6195785364d, type: 3}
m_Name:
m_EditorClassIdentifier:
feedbackText: {fileID: 5233312448025626833}
feedbackProgress: {fileID: 5233312447201393291}
feedbackProgressImage: {fileID: 5233312447919013134}
signPredictor: {fileID: 1991376311}
themelist: {fileID: 11400000, guid: a247e2ce790f0f746a3bc521e6ab7d58, type: 2}
letterPrefab: {fileID: 4639383499500021565, guid: c3e66e8957864914cb022af914df6a28, type: 3}
letterContainer: {fileID: 1870283439}
@@ -3550,7 +3554,6 @@ MonoBehaviour:
Scoreboard: {fileID: 1007532375}
EntriesGrid: {fileID: 1391137944}
scoreboardEntry: {fileID: 9154151134820372555, guid: d4a3a228b08d61847acc6da35b44e52c, type: 3}
feedback: {fileID: 5233312447513285388}
gottogamebutton: {fileID: 1581633295}
--- !u!1001 &1290865991
PrefabInstance:
@@ -6246,7 +6249,6 @@ MonoBehaviour:
m_Name:
m_EditorClassIdentifier:
modelList: {fileID: 11400000, guid: 39516e4e6e56f0f4f80647d9c4d8034c, type: 2}
model: {fileID: 5022602860645237092, guid: e6d85df707405ad4f97c23b07227ee99, type: 3}
modelInfoFile: {fileID: 4900000, guid: fb8b51022bdcd654a9f29c054832a1b5, type: 3}
configAsset: {fileID: 4900000, guid: 6288c43cdca97374782dac1ea87aa029, type: 3}
screen: {fileID: 1649505745}
@@ -6264,7 +6266,7 @@ Transform:
m_ConstrainProportionsScale: 0
m_Children: []
m_Father: {fileID: 0}
m_RootOrder: 5
m_RootOrder: 2
m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
--- !u!1 &2001212056
GameObject:
@@ -6574,7 +6576,7 @@ MonoBehaviour:
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 5233312447513285390}
m_Enabled: 1
m_Enabled: 0
m_EditorHideFlags: 0
m_Script: {fileID: 11500000, guid: 44e682a32ee15cc489bf50f3a06f717b, type: 3}
m_Name:
@@ -6598,7 +6600,7 @@ RectTransform:
- {fileID: 5233312448025626847}
- {fileID: 5233312447201393292}
m_Father: {fileID: 0}
m_RootOrder: 3
m_RootOrder: 4
m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
m_AnchorMin: {x: 0.5, y: 0}
m_AnchorMax: {x: 0.5, y: 0}

143
Assets/Hangman/Scripts/HangmanGameController.cs
View File

@@ -1,11 +1,13 @@
using System;
using System.Collections;
using System.Collections.Generic;
using System.Linq;
using TMPro;
using UnityEngine;
using UnityEngine.UI;
using DigitalRuby.Tween;
public class HangmanGameController : MonoBehaviour
public class HangmanGameController : AbstractFeedback
{
/// <summary>
/// The scriptable with all the themes, will be used to select a random word for hangman.
@@ -200,11 +202,6 @@ public class HangmanGameController : MonoBehaviour
/// </summary>
public GameObject scoreboardEntry;
/// <summary>
/// Accuracy feeback object
/// </summary>
public Feedback feedback;
/// <summary>
/// The button to go into the game
/// </summary>
@@ -217,6 +214,16 @@ public class HangmanGameController : MonoBehaviour
// Start is called before the first frame update
void Start()
{
StartController();
signPredictor.SetModel(ModelIndex.FINGERSPELLING);
AddSelfAsListener();
}
/// <summary>
/// Called at the start of the scene AND when the scene is replayed
/// </summary>
public void StartController()
{
// Make sure the mode starts at zero
mode = 0;
@@ -240,19 +247,6 @@ public class HangmanGameController : MonoBehaviour
user.minigames.Add(progress);
}
userList.Save();
// Hangman always uses fingerspelling
feedback.signPredictor.ChangeModel(ModelIndex.FINGERSPELLING);
// Set calllbacks
feedback.getSignCallback = () =>
{
return "A";
};
feedback.predictSignCallback = (sign) =>
{
currentsign = sign;
};
}
/// <summary>
@@ -281,7 +275,7 @@ public class HangmanGameController : MonoBehaviour
DeleteWord();
DisplayWord(currentWord);
replayButton.onClick.AddListener(Start);
replayButton.onClick.AddListener(StartController);
// Call to display the first image, corresponding to a clean image.
ChangeSprite();
}
@@ -352,7 +346,7 @@ public class HangmanGameController : MonoBehaviour
{
if (mode == 1)
{
if (currentsign != "")
if (currentsign != "" && currentsign != null)
{
char letter = currentsign.ToLower()[0];
currentsign = "";
@@ -393,7 +387,7 @@ public class HangmanGameController : MonoBehaviour
// For the first input char given by the user, check if the letter is in the word that needs to be spelled.
// Check to make sure the inputfield is not empty
if (currentsign != "")
if (currentsign != null && currentsign != "")
{
char firstLetter = currentsign.ToLower()[0];
currentsign = "";
@@ -543,21 +537,6 @@ public class HangmanGameController : MonoBehaviour
}
}
/// <summary>
/// Randomly shuffle the list of words
/// </summary>
private void ShuffleWords()
{
for (int i = words.Length - 1; i > 0; i--)
{
// Generate a random index between 0 and i (inclusive)
int j = UnityEngine.Random.Range(0, i + 1);
// Swap the values at indices i and j
(words[j], words[i]) = (words[i], words[j]);
}
}
/// <summary>
/// Update and save the scores
/// </summary>
@@ -731,4 +710,94 @@ public class HangmanGameController : MonoBehaviour
rank++;
}
}
protected override IEnumerator UpdateFeedback()
{
// Get current sign
string currentSign = "A";
// Get the predicted sign
if (signPredictor != null && signPredictor.learnableProbabilities != null &&
currentSign != null && signPredictor.learnableProbabilities.ContainsKey(currentSign))
{
float accuracy = signPredictor.learnableProbabilities[currentSign];
if (feedbackText != null && feedbackProgressImage != null)
{
if (accuracy > 0.90)
{
feedbackText.text = "Goed";
feedbackText.color = Color.green;
feedbackProgressImage.color = Color.green;
}
else if (accuracy > 0.80)
{
feedbackText.text = "Bijna...";
Color col = new Color(0xff / 255.0f, 0x66 / 255.0f, 0x00 / 255.0f);
feedbackText.color = col;
feedbackProgressImage.color = col;
}
else
{
feedbackText.text = "Detecteren...";
feedbackText.color = Color.red;
feedbackProgressImage.color = Color.red;
}
float oldValue = feedbackProgress.value;
// use an exponential scale
float newValue = Mathf.Exp(4 * (accuracy - 1.0f));
feedbackProgress.gameObject.Tween("FeedbackUpdate", oldValue, newValue, 0.2f, TweenScaleFunctions.CubicEaseInOut, (t) =>
{
if (feedbackProgress != null)
{
feedbackProgress.value = t.CurrentValue;
}
});
}
// Check whether (in)correct sign has high accuracy
foreach (var kv in signPredictor.learnableProbabilities)
{
if (kv.Value > 0.90)
{
predictedSign = kv.Key;
// Correct sign
if (predictedSign == currentSign)
{
yield return new WaitForSeconds(1.0f);
CheckEquality(predictedSign);
timer = DateTime.Now;
predictedSign = null;
previousIncorrectSign = null;
}
// Incorrect sign
else
{
if (previousIncorrectSign != predictedSign)
{
timer = DateTime.Now;
previousIncorrectSign = predictedSign;
}
else if (DateTime.Now - timer > TimeSpan.FromSeconds(2.0f))
{
CheckEquality(predictedSign);
timer = DateTime.Now;
predictedSign = null;
previousIncorrectSign = null;
}
}
break;
}
}
}
else if (feedbackProgress != null)
{
feedbackProgress.value = 0.0f;
}
}
private void CheckEquality(string sign)
{
currentsign = sign;
}
}

8
Assets/MediaPipeUnity/Interfaces.meta Normal file
View File

@@ -0,0 +1,8 @@
fileFormatVersion: 2
guid: 73c615986873dc246893879daf74c05d
folderAsset: yes
DefaultImporter:
externalObjects: {}
userData:
assetBundleName:
assetBundleVariant:

8
Assets/MediaPipeUnity/Interfaces/Listener.cs Normal file
View File

@@ -0,0 +1,8 @@
using System.Collections;
using System.Collections.Generic;
using UnityEngine;
public interface Listener
{
public void ProcessIncomingCall();
}

2
Assets/MediaPipeUnity/Scripts/Feedback.cs.meta → Assets/MediaPipeUnity/Interfaces/Listener.cs.meta
View File

@@ -1,5 +1,5 @@
fileFormatVersion: 2
guid: 44e682a32ee15cc489bf50f3a06f717b
guid: e4c1da9896d9ba2449549a016b5fd15e
MonoImporter:
externalObjects: {}
serializedVersion: 2

14
Assets/MediaPipeUnity/Interfaces/SignPredictorInterfaces.asmdef Normal file
View File

@@ -0,0 +1,14 @@
{
"name": "SignPredictorInterfaces",
"rootNamespace": "",
"references": [],
"includePlatforms": [],
"excludePlatforms": [],
"allowUnsafeCode": false,
"overrideReferences": false,
"precompiledReferences": [],
"autoReferenced": true,
"defineConstraints": [],
"versionDefines": [],
"noEngineReferences": false
}

7
Assets/MediaPipeUnity/Interfaces/SignPredictorInterfaces.asmdef.meta Normal file
View File

@@ -0,0 +1,7 @@
fileFormatVersion: 2
guid: f55a02e98b01bc849b30d9650ccd8f15
AssemblyDefinitionImporter:
externalObjects: {}
userData:
assetBundleName:
assetBundleVariant:

17
Assets/MediaPipeUnity/Prefabs/Feedback.prefab
View File

@@ -9,7 +9,6 @@ GameObject:
serializedVersion: 6
m_Component:
- component: {fileID: 4318122119930585316}
- component: {fileID: 4318122119930585317}
m_Layer: 5
m_Name: Feedback
m_TagString: Untagged
@@ -39,22 +38,6 @@ RectTransform:
m_AnchoredPosition: {x: 0, y: 200}
m_SizeDelta: {x: 500, y: 150}
m_Pivot: {x: 0.5, y: 0}
--- !u!114 &4318122119930585317
MonoBehaviour:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 4318122119930585319}
m_Enabled: 1
m_EditorHideFlags: 0
m_Script: {fileID: 11500000, guid: 44e682a32ee15cc489bf50f3a06f717b, type: 3}
m_Name:
m_EditorClassIdentifier:
feedbackText: {fileID: 4318122120222767928}
feedbackProgress: {fileID: 4318122119968934242}
feedbackProgressImage: {fileID: 4318122120334233319}
signPredictor: {fileID: 0}
--- !u!1 &4318122119968934244
GameObject:
m_ObjectHideFlags: 0

64
Assets/MediaPipeUnity/Scripts/AbstractFeedback.cs Normal file
View File

@@ -0,0 +1,64 @@
using DigitalRuby.Tween;
using Mediapipe.Unity.Tutorial;
using System;
using System.Collections;
using TMPro;
using UnityEngine;
using UnityEngine.Events;
using UnityEngine.UI;
/// <summary>
/// Class to display feedback during a course
/// </summary>
public abstract class AbstractFeedback : MonoBehaviour, Listener
{
/// <summary>
/// Reference to the feedback field
/// </summary>
public TMP_Text feedbackText;
/// <summary>
/// Reference to the progress bar
/// </summary>
public Slider feedbackProgress;
/// <summary>
/// Reference to the progress bar image, so we can add fancy colors
/// </summary>
public Image feedbackProgressImage;
/// <summary>
/// Reference to the sign predictor
/// </summary>
public SignPredictor signPredictor;
/// <summary>
/// Timer to keep track of how long a incorrect sign is performed
/// </summary>
protected DateTime timer;
/// <summary>
/// Current predicted sign
/// </summary>
protected string predictedSign = null;
/// <summary>
/// Previous incorrect sign, so we can keep track whether the user is wrong or the user is still changing signs
/// </summary>
protected string previousIncorrectSign = null;
public void ProcessIncomingCall()
{
//UpdateFeedback2();
StartCoroutine(UpdateFeedback());
}
public void AddSelfAsListener()
{
signPredictor.listeners.Add(this);
}
public void Empty() { }
protected abstract IEnumerator UpdateFeedback();
}

11
Assets/MediaPipeUnity/Scripts/AbstractFeedback.cs.meta Normal file
View File

@@ -0,0 +1,11 @@
fileFormatVersion: 2
guid: 7b5ac794337a54143a6e3077483d96c9
MonoImporter:
externalObjects: {}
serializedVersion: 2
defaultReferences: []
executionOrder: 0
icon: {instanceID: 0}
userData:
assetBundleName:
assetBundleVariant:

182
Assets/MediaPipeUnity/Scripts/Feedback.cs
View File

@@ -1,182 +0,0 @@
using DigitalRuby.Tween;
using Mediapipe.Unity.Tutorial;
using System;
using System.Collections;
using TMPro;
using UnityEngine;
using UnityEngine.Events;
using UnityEngine.UI;
/// <summary>
/// Class to display feedback during a course
/// </summary>
public class Feedback : MonoBehaviour
{
/// <summary>
/// Reference to the feedback field
/// </summary>
public TMP_Text feedbackText;
/// <summary>
/// Reference to the progress bar
/// </summary>
public Slider feedbackProgress;
/// <summary>
/// Reference to the progress bar image, so we can add fancy colors
/// </summary>
public Image feedbackProgressImage;
/// <summary>
/// Reference to the sign predictor
/// </summary>
public SignPredictor signPredictor;
/// <summary>
/// Callback for getting the correct sign
/// </summary>
public Func<string> getSignCallback;
/// <summary>
/// Callback to initiate the next sign
/// </summary>
public UnityAction<string> predictSignCallback;
/// <summary>
/// Timer to keep track of how long a incorrect sign is performed
/// </summary>
private DateTime timer;
/// <summary>
/// Current predicted sign
/// </summary>
private string predictedSign = null;
/// <summary>
/// Previous incorrect sign, so we can keep track whether the user is wrong or the user is still changing signs
/// </summary>
private string previousIncorrectSign = null;
/// <summary>
/// Start is called before the first frame update
/// </summary>
void Start()
{
// Start the coroutine to update the scale every 200 milliseconds
StartCoroutine(UpdateFeedback());
}
/// <summary>
/// UpdateScale updates the progress bar every 200ms, updated the feedback text, and progress bar color
/// If a high enough accuracy is detected, it will go to the next sign
/// </summary>
/// <returns></returns>
IEnumerator UpdateFeedback()
{
while (true)
{
if (getSignCallback != null && predictSignCallback != null)
{
// Get current sign
string currentSign = getSignCallback();
// Get the predicted sign
if (signPredictor != null && signPredictor.learnableProbabilities != null &&
currentSign != null && signPredictor.learnableProbabilities.ContainsKey(currentSign))
{
float accuracy = signPredictor.learnableProbabilities[currentSign];
if (feedbackText != null && feedbackProgressImage != null){
if (accuracy > 0.98)
{
// TODO: fix emojis
feedbackText.text = "✨ Perfect ✨";
Color col = new Color(0xff / 255.0f, 0xcc / 255.0f, 0x00 / 255.0f);
feedbackText.color = col;
feedbackProgressImage.color = col;
}
else if (accuracy > 0.95)
{
feedbackText.text = "Super!";
Color col = new Color(0x00 / 255.0f, 0xff / 255.0f, 0xcc / 255.0f);
feedbackText.color = col;
feedbackProgressImage.color = col;
}
else if (accuracy > 0.90)
{
feedbackText.text = "Goed";
feedbackText.color = Color.green;
feedbackProgressImage.color = Color.green;
}
else if (accuracy > 0.80)
{
feedbackText.text = "Bijna...";
Color col = new Color(0xff / 255.0f, 0x66 / 255.0f, 0x00 / 255.0f);
feedbackText.color = col;
feedbackProgressImage.color = col;
}
else
{
feedbackText.text = "Detecteren...";
feedbackText.color = Color.red;
feedbackProgressImage.color = Color.red;
}
float oldValue = feedbackProgress.value;
// use an exponential scale
float newValue = Mathf.Exp(4 * (accuracy - 1.0f));
feedbackProgress.gameObject.Tween("FeedbackUpdate", oldValue, newValue, 0.2f, TweenScaleFunctions.CubicEaseInOut, (t) =>
{
if (feedbackProgress != null)
{
feedbackProgress.value = t.CurrentValue;
}
});
}
// Check whether (in)correct sign has high accuracy
foreach (var kv in signPredictor.learnableProbabilities)
{
if (kv.Value > 0.90)
{
predictedSign = kv.Key;
// Correct sign
if (predictedSign == currentSign)
{
yield return new WaitForSeconds(1.0f);
predictSignCallback(predictedSign);
timer = DateTime.Now;
predictedSign = null;
previousIncorrectSign = null;
}
// Incorrect sign
else
{
if (previousIncorrectSign != predictedSign)
{
timer = DateTime.Now;
previousIncorrectSign = predictedSign;
}
else if (DateTime.Now - timer > TimeSpan.FromSeconds(2.0f))
{
predictSignCallback(predictedSign);
timer = DateTime.Now;
predictedSign = null;
previousIncorrectSign = null;
}
}
break;
}
}
}
else if(feedbackProgress != null)
{
feedbackProgress.value = 0.0f;
}
}
// Wait for 200 milliseconds before updating the scale again
yield return new WaitForSeconds(0.2f);
}
}
}

3
Assets/MediaPipeUnity/Scripts/SignPredictor.asmdef
View File

@@ -7,7 +7,8 @@
"GUID:04c4d86a70aa56c55a78c61f1ab1a56d",
"GUID:edc93f477bb73a743a97d6882ed330b3",
"GUID:58e104b97fb3752438ada2902a36dcbf",
"GUID:7f2d0ee6dd21e1d4eb25b71b7a749d25"
"GUID:7f2d0ee6dd21e1d4eb25b71b7a749d25",
"GUID:f55a02e98b01bc849b30d9650ccd8f15"
],
"includePlatforms": [],
"excludePlatforms": [],

19
Assets/MediaPipeUnity/Scripts/SignPredictor.cs
View File

@@ -141,6 +141,8 @@ namespace Mediapipe.Unity.Tutorial
/// </summary>
private Tensor inputTensor;
public List<Listener> listeners = new List<Listener>();
/// <summary>
/// Google Mediapipe setup & run
/// </summary>
@@ -207,7 +209,6 @@ namespace Mediapipe.Unity.Tutorial
keypointManager = new KeypointManager(modelInfoFile);
// check if model exists at path
//var model = ModelLoader.Load(Resources.Load<NNModel>("Models/Fingerspelling/model_A-L"));
worker = modelList.GetCurrentModel().CreateWorker();
@@ -215,15 +216,13 @@ namespace Mediapipe.Unity.Tutorial
StartCoroutine(SignRecognitionCoroutine());
StartCoroutine(MediapipeCoroutine());
}
public void ChangeModel(ModelIndex index)
/// <summary>
/// Called at the start of course/Minigame, will set the model before the start of SIgnPredictor is called.
/// </summary>
/// <param name="index">The index of the model to be used</param>
public void SetModel(ModelIndex index)
{
this.modelList.SetCurrentModel(index);
// If a worker already existed, we throw it out
worker?.Dispose();
// Add a new worker for the new model
worker = modelList.GetCurrentModel().CreateWorker();
}
/// <summary>
@@ -325,6 +324,10 @@ namespace Mediapipe.Unity.Tutorial
learnableProbabilities.Add(((char)(i + 65)).ToString(), softmaxedOutput2[i]);
}
//Debug.Log($"prob = [{learnableProbabilities.Aggregate(" ", (t, kv) => $"{t}{kv.Key}:{kv.Value} ")}]");
foreach(Listener listener in listeners)
{
listener.ProcessIncomingCall();
}
}
else
{

3
Assets/SpellingBee/PlayModeTests/SpellingBeePlayModeTests.asmdef
View File

@@ -7,7 +7,8 @@
"InterfacesScripts",
"Unity.TextMeshPro",
"SpellingBeeScripts",
"AccountsScripts"
"AccountsScripts",
"SignPredictor"
],
"includePlatforms": [],
"excludePlatforms": [],

40
Assets/SpellingBee/Scenes/Game.unity
View File

@@ -1720,6 +1720,10 @@ PrefabInstance:
propertyPath: m_LocalEulerAnglesHint.z
value: 0
objectReference: {fileID: 0}
- target: {fileID: 4318122119930585317, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3}
propertyPath: m_Enabled
value: 0
objectReference: {fileID: 0}
- target: {fileID: 4318122119930585317, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3}
propertyPath: signPredictor
value:
@@ -1739,15 +1743,37 @@ RectTransform:
m_CorrespondingSourceObject: {fileID: 4318122119930585316, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3}
m_PrefabInstance: {fileID: 967164043}
m_PrefabAsset: {fileID: 0}
--- !u!114 &967164045 stripped
--- !u!114 &967164046 stripped
MonoBehaviour:
m_CorrespondingSourceObject: {fileID: 4318122119930585317, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3}
m_CorrespondingSourceObject: {fileID: 4318122120334233319, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3}
m_PrefabInstance: {fileID: 967164043}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 0}
m_Enabled: 1
m_EditorHideFlags: 0
m_Script: {fileID: 11500000, guid: 44e682a32ee15cc489bf50f3a06f717b, type: 3}
m_Script: {fileID: 11500000, guid: fe87c0e1cc204ed48ad3b37840f39efc, type: 3}
m_Name:
m_EditorClassIdentifier:
--- !u!114 &967164047 stripped
MonoBehaviour:
m_CorrespondingSourceObject: {fileID: 4318122119968934242, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3}
m_PrefabInstance: {fileID: 967164043}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 0}
m_Enabled: 1
m_EditorHideFlags: 0
m_Script: {fileID: 11500000, guid: 67db9e8f0e2ae9c40bc1e2b64352a6b4, type: 3}
m_Name:
m_EditorClassIdentifier:
--- !u!114 &967164048 stripped
MonoBehaviour:
m_CorrespondingSourceObject: {fileID: 4318122120222767928, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3}
m_PrefabInstance: {fileID: 967164043}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 0}
m_Enabled: 1
m_EditorHideFlags: 0
m_Script: {fileID: 11500000, guid: f4688fdb7df04437aeb418b961361dc5, type: 3}
m_Name:
m_EditorClassIdentifier:
--- !u!1 &978093274
@@ -3144,7 +3170,7 @@ MonoBehaviour:
m_Calls:
- m_Target: {fileID: 1768150807}
m_TargetAssemblyTypeName: GameController, SpellingBeeScripts
m_MethodName: Start
m_MethodName: StartController
m_Mode: 1
m_Arguments:
m_ObjectArgument: {fileID: 0}
@@ -3901,7 +3927,6 @@ MonoBehaviour:
m_Name:
m_EditorClassIdentifier:
modelList: {fileID: 11400000, guid: 39516e4e6e56f0f4f80647d9c4d8034c, type: 2}
model: {fileID: 5022602860645237092, guid: e6d85df707405ad4f97c23b07227ee99, type: 3}
modelInfoFile: {fileID: 4900000, guid: fb8b51022bdcd654a9f29c054832a1b5, type: 3}
configAsset: {fileID: 4900000, guid: 6288c43cdca97374782dac1ea87aa029, type: 3}
screen: {fileID: 1743003084}
@@ -4553,6 +4578,10 @@ MonoBehaviour:
m_Script: {fileID: 11500000, guid: 44fbed5ae228de39b9f727def7578d06, type: 3}
m_Name:
m_EditorClassIdentifier:
feedbackText: {fileID: 967164048}
feedbackProgress: {fileID: 967164047}
feedbackProgressImage: {fileID: 967164046}
signPredictor: {fileID: 1592592444}
themeList: {fileID: 11400000, guid: a247e2ce790f0f746a3bc521e6ab7d58, type: 2}
userList: {fileID: 11400000, guid: 072bec636a40f7e4e93b0ac624a3bda2, type: 2}
minigame: {fileID: 11400000, guid: 8a087d241d652634eb4f6352267ea7dc, type: 2}
@@ -4563,7 +4592,6 @@ MonoBehaviour:
timerText: {fileID: 1843239267}
bonusTimeText: {fileID: 1812475780}
Scoreboard: {fileID: 862382568}
feedback: {fileID: 967164045}
gameEndedPanel: {fileID: 757133117}
--- !u!1 &1812475780
GameObject:

139
Assets/SpellingBee/Scripts/GameController.cs
View File

@@ -5,8 +5,9 @@ using System.Linq;
using TMPro;
using UnityEngine;
using UnityEngine.UI;
using DigitalRuby.Tween;
public partial class GameController : MonoBehaviour
public partial class GameController : AbstractFeedback
{
/// <summary>
/// All of the words that can be used in this session
@@ -136,11 +137,6 @@ public partial class GameController : MonoBehaviour
/// </summary>
public Transform Scoreboard;
/// <summary>
/// Accuracy feeback object
/// </summary>
public Feedback feedback;
/// <summary>
/// Reference to the gameEnded panel, so we can update its display
/// </summary>
@@ -150,6 +146,14 @@ public partial class GameController : MonoBehaviour
/// Start is called before the first frame update
/// </summary>
public void Start()
{
StartController();
signPredictor.SetModel(currentTheme.modelIndex);
AddSelfAsListener();
}
public void StartController()
{
correctLetters = 0;
incorrectLetters = 0;
@@ -182,29 +186,10 @@ public partial class GameController : MonoBehaviour
userList.Save();
currentTheme = minigame.themeList.themes[minigame.themeList.currentThemeIndex];
feedback.signPredictor.ChangeModel(currentTheme.modelIndex);
//feedback.signPredictor.ChangeModel(currentTheme.modelIndex);
words.AddRange(currentTheme.learnables);
ShuffleWords();
NextWord();
// Set calllbacks
feedback.getSignCallback = () =>
{
if (letterIndex < currentWord.Length)
{
return currentWord[letterIndex].ToString().ToUpper();
}
return null;
};
feedback.predictSignCallback = (sign) =>
{
bool successful = sign.ToUpper() == currentWord[letterIndex].ToString().ToUpper();
if (successful)
{
AddSeconds(secondsPerLetter);
}
NextLetter(successful);
};
}
/// <summary>
@@ -453,4 +438,106 @@ public partial class GameController : MonoBehaviour
{
yield return new WaitForSecondsRealtime(2);
}
protected override IEnumerator UpdateFeedback()
{
// Get current sign
string currentSign = GetSign();
// Get the predicted sign
if (signPredictor != null && signPredictor.learnableProbabilities != null &&
currentSign != null && signPredictor.learnableProbabilities.ContainsKey(currentSign))
{
float accuracy = signPredictor.learnableProbabilities[currentSign];
if (feedbackText != null && feedbackProgressImage != null)
{
if (accuracy > 0.90)
{
feedbackText.text = "Goed";
feedbackText.color = Color.green;
feedbackProgressImage.color = Color.green;
}
else if (accuracy > 0.80)
{
feedbackText.text = "Bijna...";
Color col = new Color(0xff / 255.0f, 0x66 / 255.0f, 0x00 / 255.0f);
feedbackText.color = col;
feedbackProgressImage.color = col;
}
else
{
feedbackText.text = "Detecteren...";
feedbackText.color = Color.red;
feedbackProgressImage.color = Color.red;
}
float oldValue = feedbackProgress.value;
// use an exponential scale
float newValue = Mathf.Exp(4 * (accuracy - 1.0f));
feedbackProgress.gameObject.Tween("FeedbackUpdate", oldValue, newValue, 0.2f, TweenScaleFunctions.CubicEaseInOut, (t) =>
{
if (feedbackProgress != null)
{
feedbackProgress.value = t.CurrentValue;
}
});
}
// Check whether (in)correct sign has high accuracy
foreach (var kv in signPredictor.learnableProbabilities)
{
if (kv.Value > 0.90)
{
predictedSign = kv.Key;
// Correct sign
if (predictedSign == currentSign)
{
//yield return new WaitForSeconds(1.0f);
predictSign(predictedSign);
timer = DateTime.Now;
predictedSign = null;
previousIncorrectSign = null;
}
// Incorrect sign
else
{
if (previousIncorrectSign != predictedSign)
{
timer = DateTime.Now;
previousIncorrectSign = predictedSign;
}
else if (DateTime.Now - timer > TimeSpan.FromSeconds(4.0f))
{
predictSign(predictedSign);
timer = DateTime.Now;
predictedSign = null;
previousIncorrectSign = null;
}
}
break;
}
}
}
else if (feedbackProgress != null)
{
feedbackProgress.value = 0.0f;
}
yield return null;
}
public string GetSign(){
if (letterIndex<currentWord.Length){
return currentWord[letterIndex].ToString().ToUpper();
}
return null;
}
public void predictSign(string sign) {
bool successful = sign.ToUpper() == currentWord[letterIndex].ToString().ToUpper();
if (successful)
{
AddSeconds(secondsPerLetter);
}
NextLetter(successful);
}
}

3
Assets/SpellingBee/Scripts/SpellingBeeScripts.asmdef
View File

@@ -7,7 +7,8 @@
"GUID:3444c67d5a3a93e5a95a48906078c372",
"GUID:d0b6b39a21908f94fbbd9f2c196a9725",
"GUID:5c2b5ba89f9e74e418232e154bc5cc7a",
"GUID:7f2d0ee6dd21e1d4eb25b71b7a749d25"
"GUID:7f2d0ee6dd21e1d4eb25b71b7a749d25",
"GUID:58e104b97fb3752438ada2902a36dcbf"
],
"includePlatforms": [],
"excludePlatforms": [],