diff --git a/Assets/Common/PlayModeTests/CommonPlayModeTests.asmdef b/Assets/Common/PlayModeTests/CommonPlayModeTests.asmdef index 9e75bf8..1682fab 100644 --- a/Assets/Common/PlayModeTests/CommonPlayModeTests.asmdef +++ b/Assets/Common/PlayModeTests/CommonPlayModeTests.asmdef @@ -8,7 +8,8 @@ "CommonScripts", "InterfacesScripts", "Unity.TextMeshPro", - "AccountsScripts" + "AccountsScripts", + "SignPredictor" ], "includePlatforms": [], "excludePlatforms": [], diff --git a/Assets/Common/Scripts/CommonScripts.asmdef b/Assets/Common/Scripts/CommonScripts.asmdef index 8b1a1ba..8cd1fb2 100644 --- a/Assets/Common/Scripts/CommonScripts.asmdef +++ b/Assets/Common/Scripts/CommonScripts.asmdef @@ -6,7 +6,8 @@ "GUID:63c63e721f65ebb7d871cb9ef49f4752", "GUID:1631ed2680c61245b8211d943c1639a8", "GUID:5c2b5ba89f9e74e418232e154bc5cc7a", - "GUID:7f2d0ee6dd21e1d4eb25b71b7a749d25" + "GUID:7f2d0ee6dd21e1d4eb25b71b7a749d25", + "GUID:d0b6b39a21908f94fbbd9f2c196a9725" ], "includePlatforms": [], "excludePlatforms": [], diff --git a/Assets/Courses/Scenes/TemplateCourse.unity b/Assets/Courses/Scenes/TemplateCourse.unity index b64e582..b682ea3 100644 --- a/Assets/Courses/Scenes/TemplateCourse.unity +++ b/Assets/Courses/Scenes/TemplateCourse.unity @@ -490,6 +490,10 @@ MonoBehaviour: m_Script: {fileID: 11500000, guid: 6b3f784c065813a4a8364b1299284816, type: 3} m_Name: m_EditorClassIdentifier: + feedbackText: {fileID: 4318122121437849762} + feedbackProgress: {fileID: 4318122121437849761} + feedbackProgressImage: {fileID: 4318122121437849760} + signPredictor: {fileID: 883853268} previewModel: {fileID: 5022602860645237092, guid: e6d85df707405ad4f97c23b07227ee99, type: 3} feedbackProgressBar: {fileID: 4318122121437849759} previewMessage: {fileID: 2070775951} @@ -505,7 +509,6 @@ MonoBehaviour: ResultsDecription: {fileID: 100123246} CoursesButton: {fileID: 839294691} timeSpent: {fileID: 77614869} - feedback: {fileID: 1714882683} --- !u!1 &361280475 GameObject: m_ObjectHideFlags: 0 @@ -1222,8 +1225,8 @@ MonoBehaviour: m_OnClick: m_PersistentCalls: m_Calls: - - m_Target: {fileID: 301088551} - m_TargetAssemblyTypeName: TemplateCourse, Assembly-CSharp + - m_Target: {fileID: 1335886461} + m_TargetAssemblyTypeName: BackButton, CommonScripts m_MethodName: Back m_Mode: 1 m_Arguments: @@ -1302,7 +1305,6 @@ MonoBehaviour: m_Name: m_EditorClassIdentifier: modelList: {fileID: 11400000, guid: 39516e4e6e56f0f4f80647d9c4d8034c, type: 2} - model: {fileID: 5022602860645237092, guid: e6d85df707405ad4f97c23b07227ee99, type: 3} modelInfoFile: {fileID: 4900000, guid: fb8b51022bdcd654a9f29c054832a1b5, type: 3} configAsset: {fileID: 4900000, guid: 6288c43cdca97374782dac1ea87aa029, type: 3} screen: {fileID: 378145456} @@ -1840,6 +1842,17 @@ RectTransform: m_CorrespondingSourceObject: {fileID: 8299246693487308515, guid: 3bccdf365a4fbea4d8fa1aa461d3dc5c, type: 3} m_PrefabInstance: {fileID: 1335886459} m_PrefabAsset: {fileID: 0} +--- !u!114 &1335886461 stripped +MonoBehaviour: + m_CorrespondingSourceObject: {fileID: 4518652150503380115, guid: 3bccdf365a4fbea4d8fa1aa461d3dc5c, type: 3} + m_PrefabInstance: {fileID: 1335886459} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 0} + m_Enabled: 1 + m_EditorHideFlags: 0 + m_Script: {fileID: 11500000, guid: c3dd279b546423e4a8a1b28819a6c4a1, type: 3} + m_Name: + m_EditorClassIdentifier: --- !u!1 &1383144366 GameObject: m_ObjectHideFlags: 0 @@ -2253,17 +2266,6 @@ RectTransform: m_CorrespondingSourceObject: {fileID: 4318122119930585316, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3} m_PrefabInstance: {fileID: 4318122121437849758} m_PrefabAsset: {fileID: 0} ---- !u!114 &1714882683 stripped -MonoBehaviour: - m_CorrespondingSourceObject: {fileID: 4318122119930585317, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3} - m_PrefabInstance: {fileID: 4318122121437849758} - m_PrefabAsset: {fileID: 0} - m_GameObject: {fileID: 4318122121437849759} - m_Enabled: 1 - m_EditorHideFlags: 0 - m_Script: {fileID: 11500000, guid: 44e682a32ee15cc489bf50f3a06f717b, type: 3} - m_Name: - m_EditorClassIdentifier: --- !u!1 &1773033262 GameObject: m_ObjectHideFlags: 0 @@ -2839,6 +2841,10 @@ PrefabInstance: propertyPath: m_LocalEulerAnglesHint.z value: 0 objectReference: {fileID: 0} + - target: {fileID: 4318122119930585317, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3} + propertyPath: m_Enabled + value: 0 + objectReference: {fileID: 0} - target: {fileID: 4318122119930585317, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3} propertyPath: signPredictor value: @@ -2866,3 +2872,36 @@ GameObject: m_CorrespondingSourceObject: {fileID: 4318122119930585319, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3} m_PrefabInstance: {fileID: 4318122121437849758} m_PrefabAsset: {fileID: 0} +--- !u!114 &4318122121437849760 stripped +MonoBehaviour: + m_CorrespondingSourceObject: {fileID: 4318122120334233319, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3} + m_PrefabInstance: {fileID: 4318122121437849758} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 0} + m_Enabled: 1 + m_EditorHideFlags: 0 + m_Script: {fileID: 11500000, guid: fe87c0e1cc204ed48ad3b37840f39efc, type: 3} + m_Name: + m_EditorClassIdentifier: +--- !u!114 &4318122121437849761 stripped +MonoBehaviour: + m_CorrespondingSourceObject: {fileID: 4318122119968934242, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3} + m_PrefabInstance: {fileID: 4318122121437849758} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 0} + m_Enabled: 1 + m_EditorHideFlags: 0 + m_Script: {fileID: 11500000, guid: 67db9e8f0e2ae9c40bc1e2b64352a6b4, type: 3} + m_Name: + m_EditorClassIdentifier: +--- !u!114 &4318122121437849762 stripped +MonoBehaviour: + m_CorrespondingSourceObject: {fileID: 4318122120222767928, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3} + m_PrefabInstance: {fileID: 4318122121437849758} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 0} + m_Enabled: 1 + m_EditorHideFlags: 0 + m_Script: {fileID: 11500000, guid: f4688fdb7df04437aeb418b961361dc5, type: 3} + m_Name: + m_EditorClassIdentifier: diff --git a/Assets/Courses/Scripts/CourseScripts.asmdef b/Assets/Courses/Scripts/CourseScripts.asmdef index 40b6489..344749b 100644 --- a/Assets/Courses/Scripts/CourseScripts.asmdef +++ b/Assets/Courses/Scripts/CourseScripts.asmdef @@ -6,7 +6,9 @@ "AccountsScripts", "InterfacesScripts", "SignPredictor", - "Unity.Barracuda" + "Unity.Barracuda", + "Tween", + "SignPredictorInterfaces" ], "includePlatforms": [], "excludePlatforms": [], diff --git a/Assets/Courses/Scripts/TemplateCourse.cs b/Assets/Courses/Scripts/TemplateCourse.cs index 19b417a..d41227c 100644 --- a/Assets/Courses/Scripts/TemplateCourse.cs +++ b/Assets/Courses/Scripts/TemplateCourse.cs @@ -1,14 +1,16 @@ using System; +using System.Collections; using TMPro; using Unity.Barracuda; using UnityEngine; using UnityEngine.UI; using UnityEngine.Video; +using DigitalRuby.Tween; /// /// TemplateCourse scene manager /// -public class TemplateCourse : MonoBehaviour +public class TemplateCourse : AbstractFeedback { // vvv TEMPORARY STUFF vvv public NNModel previewModel; @@ -112,11 +114,6 @@ public class TemplateCourse : MonoBehaviour /// public TMP_Text timeSpent; - /// - /// Reference to the feedback script on the Feedback prefab - /// - public Feedback feedback; - /// /// This function is called when the script is initialised. /// It inactivatis the popup, finds a webcam to use and links it via the WebcamTexture to the display RawImage. @@ -124,18 +121,23 @@ public class TemplateCourse : MonoBehaviour /// Then it checks whether or not the User has started the course yet, to possibly create a new progress atribute for the course. /// Then it sets up the course-screen to display relevant information from the course-scriptable. /// - void Awake() + void Start() + { + StartGameController(); + + signPredictor.SetModel(course.theme.modelIndex); + AddSelfAsListener(); + } + + public void StartGameController() { // Setting up course course = courselist.courses[courselist.currentCourseIndex]; - feedback.signPredictor.ChangeModel(course.theme.modelIndex); maxWords = course.theme.learnables.Count; // vvv TEMPORARY STUFF vvv feedbackProgressBar.SetActive(course.theme.modelIndex != ModelIndex.NONE); previewMessage.SetActive(course.theme.modelIndex == ModelIndex.NONE); - // Instead, the NONE-modelIndex points to Fingerspelling, which gives the same result - //feedback.signPredictor.model = previewModel; // ^^^ TEMPORARY STUFF ^^^ // Create entry in current user for keeping track of progress @@ -164,23 +166,6 @@ public class TemplateCourse : MonoBehaviour ResultPanel.SetActive(false); // Set the startTime startMoment = DateTime.Now; - - // Set callbacks - feedback.getSignCallback = () => - { - if (currentWordIndex < course.theme.learnables.Count) - { - return course.theme.learnables[currentWordIndex].name; - } - return null; - }; - feedback.predictSignCallback = (sign) => - { - if (sign == course.theme.learnables[currentWordIndex].name) - { - NextSign(); - } - }; } /// @@ -287,4 +272,98 @@ public class TemplateCourse : MonoBehaviour progress.AddOrUpdate("courseProgress", 1f); userList.Save(); } + + protected override IEnumerator UpdateFeedback() + { + // Get current sign + string currentSign = course.theme.learnables[currentWordIndex].name; + // Get the predicted sign + if (signPredictor != null && signPredictor.learnableProbabilities != null && + currentSign != null && signPredictor.learnableProbabilities.ContainsKey(currentSign)) + { + float accuracy = signPredictor.learnableProbabilities[currentSign]; + if (feedbackText != null && feedbackProgressImage != null) + { + if (accuracy > 0.90) + { + feedbackText.text = "Goed"; + feedbackText.color = Color.green; + feedbackProgressImage.color = Color.green; + } + else if (accuracy > 0.80) + { + feedbackText.text = "Bijna..."; + Color col = new Color(0xff / 255.0f, 0x66 / 255.0f, 0x00 / 255.0f); + feedbackText.color = col; + feedbackProgressImage.color = col; + } + else + { + feedbackText.text = "Detecteren..."; + feedbackText.color = Color.red; + feedbackProgressImage.color = Color.red; + } + + float oldValue = feedbackProgress.value; + // use an exponential scale + float newValue = Mathf.Exp(4 * (accuracy - 1.0f)); + feedbackProgress.gameObject.Tween("FeedbackUpdate", oldValue, newValue, 0.2f, TweenScaleFunctions.CubicEaseInOut, (t) => + { + if (feedbackProgress != null) + { + feedbackProgress.value = t.CurrentValue; + } + }); + } + + // Check whether (in)correct sign has high accuracy + foreach (var kv in signPredictor.learnableProbabilities) + { + if (kv.Value > 0.90) + { + predictedSign = kv.Key; + // Correct sign + if (predictedSign == currentSign) + { + yield return new WaitForSeconds(1.0f); + CheckEquality(predictedSign); + timer = DateTime.Now; + predictedSign = null; + previousIncorrectSign = null; + } + // Incorrect sign + else + { + if (previousIncorrectSign != predictedSign) + { + timer = DateTime.Now; + previousIncorrectSign = predictedSign; + } + else if (DateTime.Now - timer > TimeSpan.FromSeconds(2.0f)) + { + CheckEquality(predictedSign); + timer = DateTime.Now; + predictedSign = null; + previousIncorrectSign = null; + } + } + break; + } + } + } + else if (feedbackProgress != null) + { + + feedbackProgress.value = 0.0f; + } + + } + + private void CheckEquality(string predicted) + { + if(predicted == course.theme.learnables[currentWordIndex].name) + { + NextSign(); + } + } } diff --git a/Assets/Hangman/Scenes/Hangman.unity b/Assets/Hangman/Scenes/Hangman.unity index 073a6fb..55011fb 100644 --- a/Assets/Hangman/Scenes/Hangman.unity +++ b/Assets/Hangman/Scenes/Hangman.unity @@ -2570,7 +2570,7 @@ Transform: m_ConstrainProportionsScale: 0 m_Children: [] m_Father: {fileID: 0} - m_RootOrder: 4 + m_RootOrder: 5 m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} --- !u!1 &896343212 GameObject: @@ -3507,7 +3507,7 @@ RectTransform: - {fileID: 1892638588} - {fileID: 56162990} m_Father: {fileID: 0} - m_RootOrder: 2 + m_RootOrder: 3 m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} m_AnchorMin: {x: 0, y: 0} m_AnchorMax: {x: 0, y: 0} @@ -3526,6 +3526,10 @@ MonoBehaviour: m_Script: {fileID: 11500000, guid: 2db44635e0eb1e9429a2e6195785364d, type: 3} m_Name: m_EditorClassIdentifier: + feedbackText: {fileID: 5233312448025626833} + feedbackProgress: {fileID: 5233312447201393291} + feedbackProgressImage: {fileID: 5233312447919013134} + signPredictor: {fileID: 1991376311} themelist: {fileID: 11400000, guid: a247e2ce790f0f746a3bc521e6ab7d58, type: 2} letterPrefab: {fileID: 4639383499500021565, guid: c3e66e8957864914cb022af914df6a28, type: 3} letterContainer: {fileID: 1870283439} @@ -3550,7 +3554,6 @@ MonoBehaviour: Scoreboard: {fileID: 1007532375} EntriesGrid: {fileID: 1391137944} scoreboardEntry: {fileID: 9154151134820372555, guid: d4a3a228b08d61847acc6da35b44e52c, type: 3} - feedback: {fileID: 5233312447513285388} gottogamebutton: {fileID: 1581633295} --- !u!1001 &1290865991 PrefabInstance: @@ -6246,7 +6249,6 @@ MonoBehaviour: m_Name: m_EditorClassIdentifier: modelList: {fileID: 11400000, guid: 39516e4e6e56f0f4f80647d9c4d8034c, type: 2} - model: {fileID: 5022602860645237092, guid: e6d85df707405ad4f97c23b07227ee99, type: 3} modelInfoFile: {fileID: 4900000, guid: fb8b51022bdcd654a9f29c054832a1b5, type: 3} configAsset: {fileID: 4900000, guid: 6288c43cdca97374782dac1ea87aa029, type: 3} screen: {fileID: 1649505745} @@ -6264,7 +6266,7 @@ Transform: m_ConstrainProportionsScale: 0 m_Children: [] m_Father: {fileID: 0} - m_RootOrder: 5 + m_RootOrder: 2 m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} --- !u!1 &2001212056 GameObject: @@ -6574,7 +6576,7 @@ MonoBehaviour: m_PrefabInstance: {fileID: 0} m_PrefabAsset: {fileID: 0} m_GameObject: {fileID: 5233312447513285390} - m_Enabled: 1 + m_Enabled: 0 m_EditorHideFlags: 0 m_Script: {fileID: 11500000, guid: 44e682a32ee15cc489bf50f3a06f717b, type: 3} m_Name: @@ -6598,7 +6600,7 @@ RectTransform: - {fileID: 5233312448025626847} - {fileID: 5233312447201393292} m_Father: {fileID: 0} - m_RootOrder: 3 + m_RootOrder: 4 m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} m_AnchorMin: {x: 0.5, y: 0} m_AnchorMax: {x: 0.5, y: 0} diff --git a/Assets/Hangman/Scripts/HangmanGameController.cs b/Assets/Hangman/Scripts/HangmanGameController.cs index 9771fc6..561303e 100644 --- a/Assets/Hangman/Scripts/HangmanGameController.cs +++ b/Assets/Hangman/Scripts/HangmanGameController.cs @@ -1,11 +1,13 @@ using System; +using System.Collections; using System.Collections.Generic; using System.Linq; using TMPro; using UnityEngine; using UnityEngine.UI; +using DigitalRuby.Tween; -public class HangmanGameController : MonoBehaviour +public class HangmanGameController : AbstractFeedback { /// /// The scriptable with all the themes, will be used to select a random word for hangman. @@ -200,11 +202,6 @@ public class HangmanGameController : MonoBehaviour /// public GameObject scoreboardEntry; - /// - /// Accuracy feeback object - /// - public Feedback feedback; - /// /// The button to go into the game /// @@ -217,6 +214,16 @@ public class HangmanGameController : MonoBehaviour // Start is called before the first frame update void Start() + { + StartController(); + + signPredictor.SetModel(ModelIndex.FINGERSPELLING); + AddSelfAsListener(); + } + /// + /// Called at the start of the scene AND when the scene is replayed + /// + public void StartController() { // Make sure the mode starts at zero mode = 0; @@ -240,19 +247,6 @@ public class HangmanGameController : MonoBehaviour user.minigames.Add(progress); } userList.Save(); - - // Hangman always uses fingerspelling - feedback.signPredictor.ChangeModel(ModelIndex.FINGERSPELLING); - - // Set calllbacks - feedback.getSignCallback = () => - { - return "A"; - }; - feedback.predictSignCallback = (sign) => - { - currentsign = sign; - }; } /// @@ -281,7 +275,7 @@ public class HangmanGameController : MonoBehaviour DeleteWord(); DisplayWord(currentWord); - replayButton.onClick.AddListener(Start); + replayButton.onClick.AddListener(StartController); // Call to display the first image, corresponding to a clean image. ChangeSprite(); } @@ -352,7 +346,7 @@ public class HangmanGameController : MonoBehaviour { if (mode == 1) { - if (currentsign != "") + if (currentsign != "" && currentsign != null) { char letter = currentsign.ToLower()[0]; currentsign = ""; @@ -393,7 +387,7 @@ public class HangmanGameController : MonoBehaviour // For the first input char given by the user, check if the letter is in the word that needs to be spelled. // Check to make sure the inputfield is not empty - if (currentsign != "") + if (currentsign != null && currentsign != "") { char firstLetter = currentsign.ToLower()[0]; currentsign = ""; @@ -543,21 +537,6 @@ public class HangmanGameController : MonoBehaviour } } - /// - /// Randomly shuffle the list of words - /// - private void ShuffleWords() - { - for (int i = words.Length - 1; i > 0; i--) - { - // Generate a random index between 0 and i (inclusive) - int j = UnityEngine.Random.Range(0, i + 1); - - // Swap the values at indices i and j - (words[j], words[i]) = (words[i], words[j]); - } - } - /// /// Update and save the scores /// @@ -731,4 +710,94 @@ public class HangmanGameController : MonoBehaviour rank++; } } + + protected override IEnumerator UpdateFeedback() + { + // Get current sign + string currentSign = "A"; + // Get the predicted sign + if (signPredictor != null && signPredictor.learnableProbabilities != null && + currentSign != null && signPredictor.learnableProbabilities.ContainsKey(currentSign)) + { + float accuracy = signPredictor.learnableProbabilities[currentSign]; + if (feedbackText != null && feedbackProgressImage != null) + { + if (accuracy > 0.90) + { + feedbackText.text = "Goed"; + feedbackText.color = Color.green; + feedbackProgressImage.color = Color.green; + } + else if (accuracy > 0.80) + { + feedbackText.text = "Bijna..."; + Color col = new Color(0xff / 255.0f, 0x66 / 255.0f, 0x00 / 255.0f); + feedbackText.color = col; + feedbackProgressImage.color = col; + } + else + { + feedbackText.text = "Detecteren..."; + feedbackText.color = Color.red; + feedbackProgressImage.color = Color.red; + } + + float oldValue = feedbackProgress.value; + // use an exponential scale + float newValue = Mathf.Exp(4 * (accuracy - 1.0f)); + feedbackProgress.gameObject.Tween("FeedbackUpdate", oldValue, newValue, 0.2f, TweenScaleFunctions.CubicEaseInOut, (t) => + { + if (feedbackProgress != null) + { + feedbackProgress.value = t.CurrentValue; + } + }); + } + + // Check whether (in)correct sign has high accuracy + foreach (var kv in signPredictor.learnableProbabilities) + { + if (kv.Value > 0.90) + { + predictedSign = kv.Key; + // Correct sign + if (predictedSign == currentSign) + { + yield return new WaitForSeconds(1.0f); + CheckEquality(predictedSign); + timer = DateTime.Now; + predictedSign = null; + previousIncorrectSign = null; + } + // Incorrect sign + else + { + if (previousIncorrectSign != predictedSign) + { + timer = DateTime.Now; + previousIncorrectSign = predictedSign; + } + else if (DateTime.Now - timer > TimeSpan.FromSeconds(2.0f)) + { + CheckEquality(predictedSign); + timer = DateTime.Now; + predictedSign = null; + previousIncorrectSign = null; + } + } + break; + } + } + } + else if (feedbackProgress != null) + { + + feedbackProgress.value = 0.0f; + } + + } + private void CheckEquality(string sign) + { + currentsign = sign; + } } diff --git a/Assets/MediaPipeUnity/Interfaces.meta b/Assets/MediaPipeUnity/Interfaces.meta new file mode 100644 index 0000000..80634de --- /dev/null +++ b/Assets/MediaPipeUnity/Interfaces.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: 73c615986873dc246893879daf74c05d +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Assets/MediaPipeUnity/Interfaces/Listener.cs b/Assets/MediaPipeUnity/Interfaces/Listener.cs new file mode 100644 index 0000000..644a1e7 --- /dev/null +++ b/Assets/MediaPipeUnity/Interfaces/Listener.cs @@ -0,0 +1,8 @@ +using System.Collections; +using System.Collections.Generic; +using UnityEngine; + +public interface Listener +{ + public void ProcessIncomingCall(); +} diff --git a/Assets/MediaPipeUnity/Scripts/Feedback.cs.meta b/Assets/MediaPipeUnity/Interfaces/Listener.cs.meta similarity index 83% rename from Assets/MediaPipeUnity/Scripts/Feedback.cs.meta rename to Assets/MediaPipeUnity/Interfaces/Listener.cs.meta index cd8d001..704b3ba 100644 --- a/Assets/MediaPipeUnity/Scripts/Feedback.cs.meta +++ b/Assets/MediaPipeUnity/Interfaces/Listener.cs.meta @@ -1,5 +1,5 @@ fileFormatVersion: 2 -guid: 44e682a32ee15cc489bf50f3a06f717b +guid: e4c1da9896d9ba2449549a016b5fd15e MonoImporter: externalObjects: {} serializedVersion: 2 diff --git a/Assets/MediaPipeUnity/Interfaces/SignPredictorInterfaces.asmdef b/Assets/MediaPipeUnity/Interfaces/SignPredictorInterfaces.asmdef new file mode 100644 index 0000000..a39877a --- /dev/null +++ b/Assets/MediaPipeUnity/Interfaces/SignPredictorInterfaces.asmdef @@ -0,0 +1,14 @@ +{ + "name": "SignPredictorInterfaces", + "rootNamespace": "", + "references": [], + "includePlatforms": [], + "excludePlatforms": [], + "allowUnsafeCode": false, + "overrideReferences": false, + "precompiledReferences": [], + "autoReferenced": true, + "defineConstraints": [], + "versionDefines": [], + "noEngineReferences": false +} \ No newline at end of file diff --git a/Assets/MediaPipeUnity/Interfaces/SignPredictorInterfaces.asmdef.meta b/Assets/MediaPipeUnity/Interfaces/SignPredictorInterfaces.asmdef.meta new file mode 100644 index 0000000..c87212f --- /dev/null +++ b/Assets/MediaPipeUnity/Interfaces/SignPredictorInterfaces.asmdef.meta @@ -0,0 +1,7 @@ +fileFormatVersion: 2 +guid: f55a02e98b01bc849b30d9650ccd8f15 +AssemblyDefinitionImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Assets/MediaPipeUnity/Prefabs/Feedback.prefab b/Assets/MediaPipeUnity/Prefabs/Feedback.prefab index ee82a36..47e40cf 100644 --- a/Assets/MediaPipeUnity/Prefabs/Feedback.prefab +++ b/Assets/MediaPipeUnity/Prefabs/Feedback.prefab @@ -9,7 +9,6 @@ GameObject: serializedVersion: 6 m_Component: - component: {fileID: 4318122119930585316} - - component: {fileID: 4318122119930585317} m_Layer: 5 m_Name: Feedback m_TagString: Untagged @@ -39,22 +38,6 @@ RectTransform: m_AnchoredPosition: {x: 0, y: 200} m_SizeDelta: {x: 500, y: 150} m_Pivot: {x: 0.5, y: 0} ---- !u!114 &4318122119930585317 -MonoBehaviour: - m_ObjectHideFlags: 0 - m_CorrespondingSourceObject: {fileID: 0} - m_PrefabInstance: {fileID: 0} - m_PrefabAsset: {fileID: 0} - m_GameObject: {fileID: 4318122119930585319} - m_Enabled: 1 - m_EditorHideFlags: 0 - m_Script: {fileID: 11500000, guid: 44e682a32ee15cc489bf50f3a06f717b, type: 3} - m_Name: - m_EditorClassIdentifier: - feedbackText: {fileID: 4318122120222767928} - feedbackProgress: {fileID: 4318122119968934242} - feedbackProgressImage: {fileID: 4318122120334233319} - signPredictor: {fileID: 0} --- !u!1 &4318122119968934244 GameObject: m_ObjectHideFlags: 0 diff --git a/Assets/MediaPipeUnity/Scripts/AbstractFeedback.cs b/Assets/MediaPipeUnity/Scripts/AbstractFeedback.cs new file mode 100644 index 0000000..95e4619 --- /dev/null +++ b/Assets/MediaPipeUnity/Scripts/AbstractFeedback.cs @@ -0,0 +1,64 @@ +using DigitalRuby.Tween; +using Mediapipe.Unity.Tutorial; +using System; +using System.Collections; +using TMPro; +using UnityEngine; +using UnityEngine.Events; +using UnityEngine.UI; + +/// +/// Class to display feedback during a course +/// +public abstract class AbstractFeedback : MonoBehaviour, Listener +{ + /// + /// Reference to the feedback field + /// + public TMP_Text feedbackText; + + /// + /// Reference to the progress bar + /// + public Slider feedbackProgress; + + /// + /// Reference to the progress bar image, so we can add fancy colors + /// + public Image feedbackProgressImage; + + /// + /// Reference to the sign predictor + /// + public SignPredictor signPredictor; + + /// + /// Timer to keep track of how long a incorrect sign is performed + /// + protected DateTime timer; + + /// + /// Current predicted sign + /// + protected string predictedSign = null; + + /// + /// Previous incorrect sign, so we can keep track whether the user is wrong or the user is still changing signs + /// + protected string previousIncorrectSign = null; + + public void ProcessIncomingCall() + { + //UpdateFeedback2(); + StartCoroutine(UpdateFeedback()); + } + + public void AddSelfAsListener() + { + signPredictor.listeners.Add(this); + } + + public void Empty() { } + + protected abstract IEnumerator UpdateFeedback(); +} \ No newline at end of file diff --git a/Assets/MediaPipeUnity/Scripts/AbstractFeedback.cs.meta b/Assets/MediaPipeUnity/Scripts/AbstractFeedback.cs.meta new file mode 100644 index 0000000..5fdbcf9 --- /dev/null +++ b/Assets/MediaPipeUnity/Scripts/AbstractFeedback.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 7b5ac794337a54143a6e3077483d96c9 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Assets/MediaPipeUnity/Scripts/Feedback.cs b/Assets/MediaPipeUnity/Scripts/Feedback.cs deleted file mode 100644 index bbb88a0..0000000 --- a/Assets/MediaPipeUnity/Scripts/Feedback.cs +++ /dev/null @@ -1,182 +0,0 @@ -using DigitalRuby.Tween; -using Mediapipe.Unity.Tutorial; -using System; -using System.Collections; -using TMPro; -using UnityEngine; -using UnityEngine.Events; -using UnityEngine.UI; - -/// -/// Class to display feedback during a course -/// -public class Feedback : MonoBehaviour -{ - /// - /// Reference to the feedback field - /// - public TMP_Text feedbackText; - - /// - /// Reference to the progress bar - /// - public Slider feedbackProgress; - - /// - /// Reference to the progress bar image, so we can add fancy colors - /// - public Image feedbackProgressImage; - - /// - /// Reference to the sign predictor - /// - public SignPredictor signPredictor; - - /// - /// Callback for getting the correct sign - /// - public Func getSignCallback; - - /// - /// Callback to initiate the next sign - /// - public UnityAction predictSignCallback; - - /// - /// Timer to keep track of how long a incorrect sign is performed - /// - private DateTime timer; - - /// - /// Current predicted sign - /// - private string predictedSign = null; - - /// - /// Previous incorrect sign, so we can keep track whether the user is wrong or the user is still changing signs - /// - private string previousIncorrectSign = null; - - /// - /// Start is called before the first frame update - /// - void Start() - { - // Start the coroutine to update the scale every 200 milliseconds - StartCoroutine(UpdateFeedback()); - } - - /// - /// UpdateScale updates the progress bar every 200ms, updated the feedback text, and progress bar color - /// If a high enough accuracy is detected, it will go to the next sign - /// - /// - IEnumerator UpdateFeedback() - { - while (true) - { - if (getSignCallback != null && predictSignCallback != null) - { - - // Get current sign - string currentSign = getSignCallback(); - // Get the predicted sign - if (signPredictor != null && signPredictor.learnableProbabilities != null && - currentSign != null && signPredictor.learnableProbabilities.ContainsKey(currentSign)) - { - float accuracy = signPredictor.learnableProbabilities[currentSign]; - if (feedbackText != null && feedbackProgressImage != null){ - if (accuracy > 0.98) - { - // TODO: fix emojis - feedbackText.text = "✨ Perfect ✨"; - Color col = new Color(0xff / 255.0f, 0xcc / 255.0f, 0x00 / 255.0f); - feedbackText.color = col; - feedbackProgressImage.color = col; - } - else if (accuracy > 0.95) - { - feedbackText.text = "Super!"; - Color col = new Color(0x00 / 255.0f, 0xff / 255.0f, 0xcc / 255.0f); - feedbackText.color = col; - feedbackProgressImage.color = col; - } - else if (accuracy > 0.90) - { - feedbackText.text = "Goed"; - feedbackText.color = Color.green; - feedbackProgressImage.color = Color.green; - } - else if (accuracy > 0.80) - { - feedbackText.text = "Bijna..."; - Color col = new Color(0xff / 255.0f, 0x66 / 255.0f, 0x00 / 255.0f); - feedbackText.color = col; - feedbackProgressImage.color = col; - } - else - { - feedbackText.text = "Detecteren..."; - feedbackText.color = Color.red; - feedbackProgressImage.color = Color.red; - } - - float oldValue = feedbackProgress.value; - // use an exponential scale - float newValue = Mathf.Exp(4 * (accuracy - 1.0f)); - feedbackProgress.gameObject.Tween("FeedbackUpdate", oldValue, newValue, 0.2f, TweenScaleFunctions.CubicEaseInOut, (t) => - { - if (feedbackProgress != null) - { - feedbackProgress.value = t.CurrentValue; - } - }); - } - - // Check whether (in)correct sign has high accuracy - foreach (var kv in signPredictor.learnableProbabilities) - { - if (kv.Value > 0.90) - { - predictedSign = kv.Key; - // Correct sign - if (predictedSign == currentSign) - { - yield return new WaitForSeconds(1.0f); - predictSignCallback(predictedSign); - timer = DateTime.Now; - predictedSign = null; - previousIncorrectSign = null; - } - // Incorrect sign - else - { - if (previousIncorrectSign != predictedSign) - { - timer = DateTime.Now; - previousIncorrectSign = predictedSign; - } - else if (DateTime.Now - timer > TimeSpan.FromSeconds(2.0f)) - { - predictSignCallback(predictedSign); - timer = DateTime.Now; - predictedSign = null; - previousIncorrectSign = null; - } - } - break; - } - } - } - else if(feedbackProgress != null) - { - - feedbackProgress.value = 0.0f; - } - } - - // Wait for 200 milliseconds before updating the scale again - yield return new WaitForSeconds(0.2f); - } - } -} \ No newline at end of file diff --git a/Assets/MediaPipeUnity/Scripts/SignPredictor.asmdef b/Assets/MediaPipeUnity/Scripts/SignPredictor.asmdef index ba94856..2a113bb 100644 --- a/Assets/MediaPipeUnity/Scripts/SignPredictor.asmdef +++ b/Assets/MediaPipeUnity/Scripts/SignPredictor.asmdef @@ -7,7 +7,8 @@ "GUID:04c4d86a70aa56c55a78c61f1ab1a56d", "GUID:edc93f477bb73a743a97d6882ed330b3", "GUID:58e104b97fb3752438ada2902a36dcbf", - "GUID:7f2d0ee6dd21e1d4eb25b71b7a749d25" + "GUID:7f2d0ee6dd21e1d4eb25b71b7a749d25", + "GUID:f55a02e98b01bc849b30d9650ccd8f15" ], "includePlatforms": [], "excludePlatforms": [], diff --git a/Assets/MediaPipeUnity/Scripts/SignPredictor.cs b/Assets/MediaPipeUnity/Scripts/SignPredictor.cs index 2ec4263..8a95555 100644 --- a/Assets/MediaPipeUnity/Scripts/SignPredictor.cs +++ b/Assets/MediaPipeUnity/Scripts/SignPredictor.cs @@ -141,6 +141,8 @@ namespace Mediapipe.Unity.Tutorial /// private Tensor inputTensor; + public List listeners = new List(); + /// /// Google Mediapipe setup & run /// @@ -207,7 +209,6 @@ namespace Mediapipe.Unity.Tutorial keypointManager = new KeypointManager(modelInfoFile); - // check if model exists at path //var model = ModelLoader.Load(Resources.Load("Models/Fingerspelling/model_A-L")); worker = modelList.GetCurrentModel().CreateWorker(); @@ -215,15 +216,13 @@ namespace Mediapipe.Unity.Tutorial StartCoroutine(SignRecognitionCoroutine()); StartCoroutine(MediapipeCoroutine()); } - - public void ChangeModel(ModelIndex index) + /// + /// Called at the start of course/Minigame, will set the model before the start of SIgnPredictor is called. + /// + /// The index of the model to be used + public void SetModel(ModelIndex index) { this.modelList.SetCurrentModel(index); - // If a worker already existed, we throw it out - worker?.Dispose(); - - // Add a new worker for the new model - worker = modelList.GetCurrentModel().CreateWorker(); } /// @@ -325,6 +324,10 @@ namespace Mediapipe.Unity.Tutorial learnableProbabilities.Add(((char)(i + 65)).ToString(), softmaxedOutput2[i]); } //Debug.Log($"prob = [{learnableProbabilities.Aggregate(" ", (t, kv) => $"{t}{kv.Key}:{kv.Value} ")}]"); + foreach(Listener listener in listeners) + { + listener.ProcessIncomingCall(); + } } else { diff --git a/Assets/SpellingBee/PlayModeTests/SpellingBeePlayModeTests.asmdef b/Assets/SpellingBee/PlayModeTests/SpellingBeePlayModeTests.asmdef index a332ab7..3cb15ac 100644 --- a/Assets/SpellingBee/PlayModeTests/SpellingBeePlayModeTests.asmdef +++ b/Assets/SpellingBee/PlayModeTests/SpellingBeePlayModeTests.asmdef @@ -7,7 +7,8 @@ "InterfacesScripts", "Unity.TextMeshPro", "SpellingBeeScripts", - "AccountsScripts" + "AccountsScripts", + "SignPredictor" ], "includePlatforms": [], "excludePlatforms": [], diff --git a/Assets/SpellingBee/Scenes/Game.unity b/Assets/SpellingBee/Scenes/Game.unity index 274be82..e5085ad 100644 --- a/Assets/SpellingBee/Scenes/Game.unity +++ b/Assets/SpellingBee/Scenes/Game.unity @@ -1720,6 +1720,10 @@ PrefabInstance: propertyPath: m_LocalEulerAnglesHint.z value: 0 objectReference: {fileID: 0} + - target: {fileID: 4318122119930585317, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3} + propertyPath: m_Enabled + value: 0 + objectReference: {fileID: 0} - target: {fileID: 4318122119930585317, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3} propertyPath: signPredictor value: @@ -1739,15 +1743,37 @@ RectTransform: m_CorrespondingSourceObject: {fileID: 4318122119930585316, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3} m_PrefabInstance: {fileID: 967164043} m_PrefabAsset: {fileID: 0} ---- !u!114 &967164045 stripped +--- !u!114 &967164046 stripped MonoBehaviour: - m_CorrespondingSourceObject: {fileID: 4318122119930585317, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3} + m_CorrespondingSourceObject: {fileID: 4318122120334233319, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3} m_PrefabInstance: {fileID: 967164043} m_PrefabAsset: {fileID: 0} m_GameObject: {fileID: 0} m_Enabled: 1 m_EditorHideFlags: 0 - m_Script: {fileID: 11500000, guid: 44e682a32ee15cc489bf50f3a06f717b, type: 3} + m_Script: {fileID: 11500000, guid: fe87c0e1cc204ed48ad3b37840f39efc, type: 3} + m_Name: + m_EditorClassIdentifier: +--- !u!114 &967164047 stripped +MonoBehaviour: + m_CorrespondingSourceObject: {fileID: 4318122119968934242, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3} + m_PrefabInstance: {fileID: 967164043} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 0} + m_Enabled: 1 + m_EditorHideFlags: 0 + m_Script: {fileID: 11500000, guid: 67db9e8f0e2ae9c40bc1e2b64352a6b4, type: 3} + m_Name: + m_EditorClassIdentifier: +--- !u!114 &967164048 stripped +MonoBehaviour: + m_CorrespondingSourceObject: {fileID: 4318122120222767928, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3} + m_PrefabInstance: {fileID: 967164043} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 0} + m_Enabled: 1 + m_EditorHideFlags: 0 + m_Script: {fileID: 11500000, guid: f4688fdb7df04437aeb418b961361dc5, type: 3} m_Name: m_EditorClassIdentifier: --- !u!1 &978093274 @@ -3144,7 +3170,7 @@ MonoBehaviour: m_Calls: - m_Target: {fileID: 1768150807} m_TargetAssemblyTypeName: GameController, SpellingBeeScripts - m_MethodName: Start + m_MethodName: StartController m_Mode: 1 m_Arguments: m_ObjectArgument: {fileID: 0} @@ -3901,7 +3927,6 @@ MonoBehaviour: m_Name: m_EditorClassIdentifier: modelList: {fileID: 11400000, guid: 39516e4e6e56f0f4f80647d9c4d8034c, type: 2} - model: {fileID: 5022602860645237092, guid: e6d85df707405ad4f97c23b07227ee99, type: 3} modelInfoFile: {fileID: 4900000, guid: fb8b51022bdcd654a9f29c054832a1b5, type: 3} configAsset: {fileID: 4900000, guid: 6288c43cdca97374782dac1ea87aa029, type: 3} screen: {fileID: 1743003084} @@ -4553,6 +4578,10 @@ MonoBehaviour: m_Script: {fileID: 11500000, guid: 44fbed5ae228de39b9f727def7578d06, type: 3} m_Name: m_EditorClassIdentifier: + feedbackText: {fileID: 967164048} + feedbackProgress: {fileID: 967164047} + feedbackProgressImage: {fileID: 967164046} + signPredictor: {fileID: 1592592444} themeList: {fileID: 11400000, guid: a247e2ce790f0f746a3bc521e6ab7d58, type: 2} userList: {fileID: 11400000, guid: 072bec636a40f7e4e93b0ac624a3bda2, type: 2} minigame: {fileID: 11400000, guid: 8a087d241d652634eb4f6352267ea7dc, type: 2} @@ -4563,7 +4592,6 @@ MonoBehaviour: timerText: {fileID: 1843239267} bonusTimeText: {fileID: 1812475780} Scoreboard: {fileID: 862382568} - feedback: {fileID: 967164045} gameEndedPanel: {fileID: 757133117} --- !u!1 &1812475780 GameObject: diff --git a/Assets/SpellingBee/Scripts/GameController.cs b/Assets/SpellingBee/Scripts/GameController.cs index d0605fc..61be88c 100644 --- a/Assets/SpellingBee/Scripts/GameController.cs +++ b/Assets/SpellingBee/Scripts/GameController.cs @@ -5,8 +5,9 @@ using System.Linq; using TMPro; using UnityEngine; using UnityEngine.UI; +using DigitalRuby.Tween; -public partial class GameController : MonoBehaviour +public partial class GameController : AbstractFeedback { /// /// All of the words that can be used in this session @@ -136,11 +137,6 @@ public partial class GameController : MonoBehaviour /// public Transform Scoreboard; - /// - /// Accuracy feeback object - /// - public Feedback feedback; - /// /// Reference to the gameEnded panel, so we can update its display /// @@ -150,6 +146,14 @@ public partial class GameController : MonoBehaviour /// Start is called before the first frame update /// public void Start() + { + StartController(); + + signPredictor.SetModel(currentTheme.modelIndex); + AddSelfAsListener(); + } + + public void StartController() { correctLetters = 0; incorrectLetters = 0; @@ -182,29 +186,10 @@ public partial class GameController : MonoBehaviour userList.Save(); currentTheme = minigame.themeList.themes[minigame.themeList.currentThemeIndex]; - feedback.signPredictor.ChangeModel(currentTheme.modelIndex); + //feedback.signPredictor.ChangeModel(currentTheme.modelIndex); words.AddRange(currentTheme.learnables); ShuffleWords(); NextWord(); - - // Set calllbacks - feedback.getSignCallback = () => - { - if (letterIndex < currentWord.Length) - { - return currentWord[letterIndex].ToString().ToUpper(); - } - return null; - }; - feedback.predictSignCallback = (sign) => - { - bool successful = sign.ToUpper() == currentWord[letterIndex].ToString().ToUpper(); - if (successful) - { - AddSeconds(secondsPerLetter); - } - NextLetter(successful); - }; } /// @@ -453,4 +438,106 @@ public partial class GameController : MonoBehaviour { yield return new WaitForSecondsRealtime(2); } + + protected override IEnumerator UpdateFeedback() + { + // Get current sign + string currentSign = GetSign(); + // Get the predicted sign + if (signPredictor != null && signPredictor.learnableProbabilities != null && + currentSign != null && signPredictor.learnableProbabilities.ContainsKey(currentSign)) + { + float accuracy = signPredictor.learnableProbabilities[currentSign]; + if (feedbackText != null && feedbackProgressImage != null) + { + if (accuracy > 0.90) + { + feedbackText.text = "Goed"; + feedbackText.color = Color.green; + feedbackProgressImage.color = Color.green; + } + else if (accuracy > 0.80) + { + feedbackText.text = "Bijna..."; + Color col = new Color(0xff / 255.0f, 0x66 / 255.0f, 0x00 / 255.0f); + feedbackText.color = col; + feedbackProgressImage.color = col; + } + else + { + feedbackText.text = "Detecteren..."; + feedbackText.color = Color.red; + feedbackProgressImage.color = Color.red; + } + + float oldValue = feedbackProgress.value; + // use an exponential scale + float newValue = Mathf.Exp(4 * (accuracy - 1.0f)); + feedbackProgress.gameObject.Tween("FeedbackUpdate", oldValue, newValue, 0.2f, TweenScaleFunctions.CubicEaseInOut, (t) => + { + if (feedbackProgress != null) + { + feedbackProgress.value = t.CurrentValue; + } + }); + } + + // Check whether (in)correct sign has high accuracy + foreach (var kv in signPredictor.learnableProbabilities) + { + if (kv.Value > 0.90) + { + predictedSign = kv.Key; + // Correct sign + if (predictedSign == currentSign) + { + //yield return new WaitForSeconds(1.0f); + predictSign(predictedSign); + timer = DateTime.Now; + predictedSign = null; + previousIncorrectSign = null; + } + // Incorrect sign + else + { + if (previousIncorrectSign != predictedSign) + { + timer = DateTime.Now; + previousIncorrectSign = predictedSign; + } + else if (DateTime.Now - timer > TimeSpan.FromSeconds(4.0f)) + { + predictSign(predictedSign); + timer = DateTime.Now; + predictedSign = null; + previousIncorrectSign = null; + } + } + break; + } + } + } + else if (feedbackProgress != null) + { + + feedbackProgress.value = 0.0f; + } + yield return null; + } + + public string GetSign(){ + if (letterIndex