diff --git a/Assets/Common/PlayModeTests/CommonPlayModeTests.asmdef b/Assets/Common/PlayModeTests/CommonPlayModeTests.asmdef index 9e75bf8..1682fab 100644 --- a/Assets/Common/PlayModeTests/CommonPlayModeTests.asmdef +++ b/Assets/Common/PlayModeTests/CommonPlayModeTests.asmdef @@ -8,7 +8,8 @@ "CommonScripts", "InterfacesScripts", "Unity.TextMeshPro", - "AccountsScripts" + "AccountsScripts", + "SignPredictor" ], "includePlatforms": [], "excludePlatforms": [], diff --git a/Assets/Common/Scripts/CommonScripts.asmdef b/Assets/Common/Scripts/CommonScripts.asmdef index 8b1a1ba..ed617d5 100644 --- a/Assets/Common/Scripts/CommonScripts.asmdef +++ b/Assets/Common/Scripts/CommonScripts.asmdef @@ -5,7 +5,6 @@ "GUID:6055be8ebefd69e48b49212b09b47b2f", "GUID:63c63e721f65ebb7d871cb9ef49f4752", "GUID:1631ed2680c61245b8211d943c1639a8", - "GUID:5c2b5ba89f9e74e418232e154bc5cc7a", "GUID:7f2d0ee6dd21e1d4eb25b71b7a749d25" ], "includePlatforms": [], diff --git a/Assets/Courses/Scenes/CourseScreen.unity b/Assets/Courses/Scenes/CourseScreen.unity index 8f81fd0..bb651dd 100644 --- a/Assets/Courses/Scenes/CourseScreen.unity +++ b/Assets/Courses/Scenes/CourseScreen.unity @@ -1273,7 +1273,6 @@ MonoBehaviour: m_Name: m_EditorClassIdentifier: modelList: {fileID: 11400000, guid: 39516e4e6e56f0f4f80647d9c4d8034c, type: 2} - model: {fileID: 5022602860645237092, guid: e6d85df707405ad4f97c23b07227ee99, type: 3} modelInfoFile: {fileID: 4900000, guid: fb8b51022bdcd654a9f29c054832a1b5, type: 3} configAsset: {fileID: 4900000, guid: 6288c43cdca97374782dac1ea87aa029, type: 3} screen: {fileID: 378145456} @@ -1529,6 +1528,7 @@ MonoBehaviour: m_Script: {fileID: 11500000, guid: 6b3f784c065813a4a8364b1299284816, type: 3} m_Name: m_EditorClassIdentifier: + signPredictor: {fileID: 883853268} previewModel: {fileID: 5022602860645237092, guid: e6d85df707405ad4f97c23b07227ee99, type: 3} feedbackProgressBar: {fileID: 4318122121437849759} previewMessage: {fileID: 2070775951} @@ -1544,7 +1544,9 @@ MonoBehaviour: ResultsDecription: {fileID: 100123246} CoursesButton: {fileID: 839294691} timeSpent: {fileID: 77614869} - feedback: {fileID: 1714882683} + feedbackText: {fileID: 4318122121437849762} + feedbackProgress: {fileID: 4318122121437849761} + feedbackProgressImage: {fileID: 4318122121437849760} --- !u!4 &1122267057 Transform: m_ObjectHideFlags: 0 @@ -2284,17 +2286,6 @@ RectTransform: m_CorrespondingSourceObject: {fileID: 4318122119930585316, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3} m_PrefabInstance: {fileID: 4318122121437849758} m_PrefabAsset: {fileID: 0} ---- !u!114 &1714882683 stripped -MonoBehaviour: - m_CorrespondingSourceObject: {fileID: 4318122119930585317, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3} - m_PrefabInstance: {fileID: 4318122121437849758} - m_PrefabAsset: {fileID: 0} - m_GameObject: {fileID: 4318122121437849759} - m_Enabled: 1 - m_EditorHideFlags: 0 - m_Script: {fileID: 11500000, guid: 44e682a32ee15cc489bf50f3a06f717b, type: 3} - m_Name: - m_EditorClassIdentifier: --- !u!1 &1773033262 GameObject: m_ObjectHideFlags: 0 @@ -2897,3 +2888,36 @@ GameObject: m_CorrespondingSourceObject: {fileID: 4318122119930585319, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3} m_PrefabInstance: {fileID: 4318122121437849758} m_PrefabAsset: {fileID: 0} +--- !u!114 &4318122121437849760 stripped +MonoBehaviour: + m_CorrespondingSourceObject: {fileID: 4318122120334233319, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3} + m_PrefabInstance: {fileID: 4318122121437849758} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 0} + m_Enabled: 1 + m_EditorHideFlags: 0 + m_Script: {fileID: 11500000, guid: fe87c0e1cc204ed48ad3b37840f39efc, type: 3} + m_Name: + m_EditorClassIdentifier: +--- !u!114 &4318122121437849761 stripped +MonoBehaviour: + m_CorrespondingSourceObject: {fileID: 4318122119968934242, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3} + m_PrefabInstance: {fileID: 4318122121437849758} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 0} + m_Enabled: 1 + m_EditorHideFlags: 0 + m_Script: {fileID: 11500000, guid: 67db9e8f0e2ae9c40bc1e2b64352a6b4, type: 3} + m_Name: + m_EditorClassIdentifier: +--- !u!114 &4318122121437849762 stripped +MonoBehaviour: + m_CorrespondingSourceObject: {fileID: 4318122120222767928, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3} + m_PrefabInstance: {fileID: 4318122121437849758} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 0} + m_Enabled: 1 + m_EditorHideFlags: 0 + m_Script: {fileID: 11500000, guid: f4688fdb7df04437aeb418b961361dc5, type: 3} + m_Name: + m_EditorClassIdentifier: diff --git a/Assets/Courses/Scripts/CourseScripts.asmdef b/Assets/Courses/Scripts/CourseScripts.asmdef index 40b6489..53c1ab0 100644 --- a/Assets/Courses/Scripts/CourseScripts.asmdef +++ b/Assets/Courses/Scripts/CourseScripts.asmdef @@ -5,8 +5,8 @@ "Unity.TextMeshPro", "AccountsScripts", "InterfacesScripts", - "SignPredictor", - "Unity.Barracuda" + "Tween", + "SignPredictor" ], "includePlatforms": [], "excludePlatforms": [], diff --git a/Assets/Courses/Scripts/CoursesController.cs b/Assets/Courses/Scripts/CoursesController.cs index 19446c7..219fc90 100644 --- a/Assets/Courses/Scripts/CoursesController.cs +++ b/Assets/Courses/Scripts/CoursesController.cs @@ -1,6 +1,7 @@ +using DigitalRuby.Tween; using System; +using System.Collections; using TMPro; -using Unity.Barracuda; using UnityEngine; using UnityEngine.UI; using UnityEngine.Video; @@ -8,13 +9,10 @@ using UnityEngine.Video; /// /// TemplateCourse scene manager /// -public class CoursesController : MonoBehaviour +public class CoursesController : AbstractFeedback { - // vvv TEMPORARY STUFF vvv - public NNModel previewModel; public GameObject feedbackProgressBar; public GameObject previewMessage; - // ^^^ TEMPORARY STUFF ^^^ /// /// Reference to instructional video player @@ -112,10 +110,37 @@ public class CoursesController : MonoBehaviour /// public TMP_Text timeSpent; + /// - /// Reference to the feedback script on the Feedback prefab + /// Reference to the feedback field /// - public Feedback feedback; + public TMP_Text feedbackText; + + /// + /// Reference to the progress bar + /// + public Slider feedbackProgress; + + /// + /// Reference to the progress bar image, so we can add fancy colors + /// + public Image feedbackProgressImage; + + /// + /// Timer to keep track of how long a incorrect sign is performed + /// + protected DateTime timer; + + /// + /// Current predicted sign + /// + protected string predictedSign = null; + + /// + /// Previous incorrect sign, so we can keep track whether the user is wrong or the user is still changing signs + /// + protected string previousIncorrectSign = null; + /// /// This function is called when the script is initialised. @@ -124,19 +149,25 @@ public class CoursesController : MonoBehaviour /// Then it checks whether or not the User has started the course yet, to possibly create a new progress atribute for the course. /// Then it sets up the course-screen to display relevant information from the course-scriptable. /// - void Awake() + void Start() + { + StartCourseController(); + + signPredictor.SetModel(course.theme.modelIndex); + AddSelfAsListener(); + } + /// + /// Holds the course-specific logic to start the controller, it is seperated to allow the course to be reset (if that would become needed) + /// + public void StartCourseController() { // Setting up course course = courselist.courses[courselist.currentCourseIndex]; - feedback.signPredictor.ChangeModel(course.theme.modelIndex); maxWords = course.theme.learnables.Count; - // vvv TEMPORARY STUFF vvv + // Show preview messages if there is no model feedbackProgressBar.SetActive(course.theme.modelIndex != ModelIndex.NONE); previewMessage.SetActive(course.theme.modelIndex == ModelIndex.NONE); - // Instead, the NONE-modelIndex points to Fingerspelling, which gives the same result - //feedback.signPredictor.model = previewModel; - // ^^^ TEMPORARY STUFF ^^^ // Create entry in current user for keeping track of progress userList.Load(); @@ -164,23 +195,6 @@ public class CoursesController : MonoBehaviour ResultPanel.SetActive(false); // Set the startTime startMoment = DateTime.Now; - - // Set callbacks - feedback.getSignCallback = () => - { - if (currentWordIndex < course.theme.learnables.Count) - { - return course.theme.learnables[currentWordIndex].name; - } - return null; - }; - feedback.predictSignCallback = (sign) => - { - if (sign == course.theme.learnables[currentWordIndex].name) - { - NextSign(); - } - }; } /// @@ -287,4 +301,104 @@ public class CoursesController : MonoBehaviour progress.AddOrUpdate("courseProgress", 1f); userList.Save(); } + /// + /// The updateFunction that is called when new probabilities become available + /// + /// + protected override IEnumerator UpdateFeedback() + { + // Get current sign + string currentSign = course.theme.learnables[currentWordIndex].name; + // Get the predicted sign + if (signPredictor != null && signPredictor.learnableProbabilities != null && + currentSign != null && signPredictor.learnableProbabilities.ContainsKey(currentSign)) + { + float accuracy = signPredictor.learnableProbabilities[currentSign]; + if (feedbackText != null && feedbackProgressImage != null) + { + if (accuracy > 0.90) + { + feedbackText.text = "Goed"; + feedbackText.color = Color.green; + feedbackProgressImage.color = Color.green; + } + else if (accuracy > 0.80) + { + feedbackText.text = "Bijna..."; + Color col = new Color(0xff / 255.0f, 0x66 / 255.0f, 0x00 / 255.0f); + feedbackText.color = col; + feedbackProgressImage.color = col; + } + else + { + feedbackText.text = "Detecteren..."; + feedbackText.color = Color.red; + feedbackProgressImage.color = Color.red; + } + + float oldValue = feedbackProgress.value; + // use an exponential scale + float newValue = Mathf.Exp(4 * (accuracy - 1.0f)); + feedbackProgress.gameObject.Tween("FeedbackUpdate", oldValue, newValue, 0.2f, TweenScaleFunctions.CubicEaseInOut, (t) => + { + if (feedbackProgress != null) + { + feedbackProgress.value = t.CurrentValue; + } + }); + } + + // Check whether (in)correct sign has high accuracy + foreach (var kv in signPredictor.learnableProbabilities) + { + if (kv.Value > 0.90) + { + predictedSign = kv.Key; + // Correct sign + if (predictedSign == currentSign) + { + yield return new WaitForSeconds(1.0f); + CheckEquality(predictedSign); + timer = DateTime.Now; + predictedSign = null; + previousIncorrectSign = null; + } + // Incorrect sign + else + { + if (previousIncorrectSign != predictedSign) + { + timer = DateTime.Now; + previousIncorrectSign = predictedSign; + } + else if (DateTime.Now - timer > TimeSpan.FromSeconds(2.0f)) + { + CheckEquality(predictedSign); + timer = DateTime.Now; + predictedSign = null; + previousIncorrectSign = null; + } + } + break; + } + } + } + else if (feedbackProgress != null) + { + + feedbackProgress.value = 0.0f; + } + + } + /// + /// Function to check equality between the current sign and the sign that the model predicted, if they are equal then the next sign is fetched. + /// + /// + private void CheckEquality(string predicted) + { + if (predicted == course.theme.learnables[currentWordIndex].name) + { + NextSign(); + } + } } diff --git a/Assets/Hangman/Scenes/HangmanGame.unity b/Assets/Hangman/Scenes/HangmanGame.unity index ccd072b..ee3a10a 100644 --- a/Assets/Hangman/Scenes/HangmanGame.unity +++ b/Assets/Hangman/Scenes/HangmanGame.unity @@ -38,7 +38,7 @@ RenderSettings: m_ReflectionIntensity: 1 m_CustomReflection: {fileID: 0} m_Sun: {fileID: 0} - m_IndirectSpecularColor: {r: 0.37311953, g: 0.38074014, b: 0.3587274, a: 1} + m_IndirectSpecularColor: {r: 0.37311918, g: 0.3807398, b: 0.35872716, a: 1} m_UseRadianceAmbientProbe: 0 --- !u!157 &3 LightmapSettings: @@ -5129,6 +5129,7 @@ MonoBehaviour: m_Script: {fileID: 11500000, guid: 2db44635e0eb1e9429a2e6195785364d, type: 3} m_Name: m_EditorClassIdentifier: + signPredictor: {fileID: 1991376311} themelist: {fileID: 11400000, guid: a247e2ce790f0f746a3bc521e6ab7d58, type: 2} letterPrefab: {fileID: 4639383499500021565, guid: c3e66e8957864914cb022af914df6a28, type: 3} letterContainer: {fileID: 1870283439} @@ -5153,8 +5154,10 @@ MonoBehaviour: Scoreboard: {fileID: 1007532375} EntriesGrid: {fileID: 1391137944} scoreboardEntry: {fileID: 9154151134820372555, guid: d4a3a228b08d61847acc6da35b44e52c, type: 3} - feedback: {fileID: 5233312447513285388} gottogamebutton: {fileID: 1581633295} + feedbackText: {fileID: 0} + feedbackProgress: {fileID: 0} + feedbackProgressImage: {fileID: 0} --- !u!1 &1678036720 GameObject: m_ObjectHideFlags: 0 @@ -6183,7 +6186,6 @@ MonoBehaviour: m_Name: m_EditorClassIdentifier: modelList: {fileID: 11400000, guid: 39516e4e6e56f0f4f80647d9c4d8034c, type: 2} - model: {fileID: 5022602860645237092, guid: e6d85df707405ad4f97c23b07227ee99, type: 3} modelInfoFile: {fileID: 4900000, guid: fb8b51022bdcd654a9f29c054832a1b5, type: 3} configAsset: {fileID: 4900000, guid: 6288c43cdca97374782dac1ea87aa029, type: 3} screen: {fileID: 1649505745} diff --git a/Assets/Hangman/Scripts/HangManWebcam.cs b/Assets/Hangman/Scripts/HangManWebcam.cs deleted file mode 100644 index 2690e54..0000000 --- a/Assets/Hangman/Scripts/HangManWebcam.cs +++ /dev/null @@ -1,62 +0,0 @@ -using System.Collections; -using System.Collections.Generic; -using UnityEngine; -using UnityEngine.UI; - -public class HangManWebcam : WebCam -{ - /// - /// The display for player 1 - /// - public RawImage display1; - - /// - /// The display for player 2 - /// - public RawImage display2; - - /// - /// We use a different awake, since we dont want the camera to start immediatelly - /// - void Awake() - { - WebCamDevice device = WebCamTexture.devices[camdex]; - tex = new WebCamTexture(device.name); - display.texture = tex; - } - - /// - /// Hangman uses two different webcam_textures, we need to be able to toggle between them - /// - public void Switch_texture() - { - if(display == display1) - { - display = display2; - } - else - { - display = display1; - } - // Give the webcamTexture to the new webcam - display.texture = tex; - } - - /// - /// Scene changing is implemented here to avoid problems with webcam - /// - public new void GotoThemeSelection() - { - //minigameList.GetIndexInMinigameList(MinigameIndex.HANGMAN); - if (tex != null) - { - if (tex.isPlaying) - { - display.texture = null; - tex.Stop(); - tex = null; - } - } - SystemController.GetInstance().BackToPreviousScene(); - } -} diff --git a/Assets/Hangman/Scripts/HangmanController.cs b/Assets/Hangman/Scripts/HangmanController.cs index f4280b2..8ce5a94 100644 --- a/Assets/Hangman/Scripts/HangmanController.cs +++ b/Assets/Hangman/Scripts/HangmanController.cs @@ -1,11 +1,13 @@ using System; +using System.Collections; using System.Collections.Generic; using System.Linq; using TMPro; using UnityEngine; using UnityEngine.UI; +using DigitalRuby.Tween; -public class HangmanController : MonoBehaviour +public class HangmanController : AbstractFeedback { /// /// The scriptable with all the themes, will be used to select a random word for hangman. @@ -18,11 +20,6 @@ public class HangmanController : MonoBehaviour /// private string currentWord; - /// - /// All of the words that can be used in this session - /// - private string[] words; - /// /// This integer holds the total amount of wrong guesses the player has made /// @@ -200,11 +197,6 @@ public class HangmanController : MonoBehaviour /// public GameObject scoreboardEntry; - /// - /// Accuracy feeback object - /// - public Feedback feedback; - /// /// The button to go into the game /// @@ -215,8 +207,50 @@ public class HangmanController : MonoBehaviour /// private String currentsign = ""; - // Start is called before the first frame update + /// + /// Reference to the feedback field + /// + public TMP_Text feedbackText; + + /// + /// Reference to the progress bar + /// + public Slider feedbackProgress; + + /// + /// Reference to the progress bar image, so we can add fancy colors + /// + public Image feedbackProgressImage; + + /// + /// Timer to keep track of how long a incorrect sign is performed + /// + protected DateTime timer; + + /// + /// Current predicted sign + /// + protected string predictedSign = null; + + /// + /// Previous incorrect sign, so we can keep track whether the user is wrong or the user is still changing signs + /// + protected string previousIncorrectSign = null; + + /// + /// Start is called before the first frame update + /// void Start() + { + StartController(); + + signPredictor.SetModel(ModelIndex.FINGERSPELLING); + AddSelfAsListener(); + } + /// + /// Called at the start of the scene AND when the scene is replayed + /// + public void StartController() { // Make sure the mode starts at zero mode = 0; @@ -240,19 +274,6 @@ public class HangmanController : MonoBehaviour user.minigames.Add(progress); } userList.Save(); - - // Hangman always uses fingerspelling - feedback.signPredictor.ChangeModel(ModelIndex.FINGERSPELLING); - - // Set calllbacks - feedback.getSignCallback = () => - { - return "A"; - }; - feedback.predictSignCallback = (sign) => - { - currentsign = sign; - }; } /// @@ -281,7 +302,7 @@ public class HangmanController : MonoBehaviour DeleteWord(); DisplayWord(currentWord); - replayButton.onClick.AddListener(Start); + replayButton.onClick.AddListener(StartController); // Call to display the first image, corresponding to a clean image. ChangeSprite(); } @@ -352,7 +373,7 @@ public class HangmanController : MonoBehaviour { if (mode == 1) { - if (currentsign != "") + if (currentsign != null && currentsign != "") { char letter = currentsign.ToLower()[0]; currentsign = ""; @@ -393,7 +414,7 @@ public class HangmanController : MonoBehaviour // For the first input char given by the user, check if the letter is in the word that needs to be spelled. // Check to make sure the inputfield is not empty - if (currentsign != "") + if (currentsign != null && currentsign != "") { char firstLetter = currentsign.ToLower()[0]; currentsign = ""; @@ -543,21 +564,6 @@ public class HangmanController : MonoBehaviour } } - /// - /// Randomly shuffle the list of words - /// - private void ShuffleWords() - { - for (int i = words.Length - 1; i > 0; i--) - { - // Generate a random index between 0 and i (inclusive) - int j = UnityEngine.Random.Range(0, i + 1); - - // Swap the values at indices i and j - (words[j], words[i]) = (words[i], words[j]); - } - } - /// /// Update and save the scores /// @@ -731,4 +737,97 @@ public class HangmanController : MonoBehaviour rank++; } } + /// + /// The updateFunction that is called when new probabilities become available + /// + /// + protected override IEnumerator UpdateFeedback() + { + // Get current sign + string currentSign = "A"; + // Get the predicted sign + if (signPredictor != null && signPredictor.learnableProbabilities != null && + currentSign != null && signPredictor.learnableProbabilities.ContainsKey(currentSign)) + { + float accuracy = signPredictor.learnableProbabilities[currentSign]; + if (feedbackText != null && feedbackProgressImage != null) + { + if (accuracy > 0.90) + { + feedbackText.text = "Goed"; + feedbackText.color = Color.green; + feedbackProgressImage.color = Color.green; + } + else if (accuracy > 0.80) + { + feedbackText.text = "Bijna..."; + Color col = new Color(0xff / 255.0f, 0x66 / 255.0f, 0x00 / 255.0f); + feedbackText.color = col; + feedbackProgressImage.color = col; + } + else + { + feedbackText.text = "Detecteren..."; + feedbackText.color = Color.red; + feedbackProgressImage.color = Color.red; + } + + float oldValue = feedbackProgress.value; + // use an exponential scale + float newValue = Mathf.Exp(4 * (accuracy - 1.0f)); + feedbackProgress.gameObject.Tween("FeedbackUpdate", oldValue, newValue, 0.2f, TweenScaleFunctions.CubicEaseInOut, (t) => + { + if (feedbackProgress != null) + { + feedbackProgress.value = t.CurrentValue; + } + }); + } + + // Check whether (in)correct sign has high accuracy + foreach (var kv in signPredictor.learnableProbabilities) + { + if (kv.Value > 0.90) + { + predictedSign = kv.Key; + // Correct sign + if (predictedSign == currentSign) + { + yield return new WaitForSeconds(1.0f); + CheckEquality(predictedSign); + timer = DateTime.Now; + predictedSign = null; + previousIncorrectSign = null; + } + // Incorrect sign + else + { + if (previousIncorrectSign != predictedSign) + { + timer = DateTime.Now; + previousIncorrectSign = predictedSign; + } + else if (DateTime.Now - timer > TimeSpan.FromSeconds(2.0f)) + { + CheckEquality(predictedSign); + timer = DateTime.Now; + predictedSign = null; + previousIncorrectSign = null; + } + } + break; + } + } + } + else if (feedbackProgress != null) + { + + feedbackProgress.value = 0.0f; + } + + } + private void CheckEquality(string sign) + { + currentsign = sign; + } } diff --git a/Assets/Hangman/Scripts/WebCam.cs b/Assets/Hangman/Scripts/WebCam.cs deleted file mode 100644 index 7809b77..0000000 --- a/Assets/Hangman/Scripts/WebCam.cs +++ /dev/null @@ -1,95 +0,0 @@ -using UnityEngine.UI; -using UnityEngine; - -public class WebCam : MonoBehaviour -{ - /// - /// Index of the current camera - /// - protected int camdex = 0; - - /// - /// Texture to paste on the display - /// - protected WebCamTexture tex; - - /// - /// Display for the video feed - /// - public RawImage display; - - /// - /// Setup the webcam correctly - /// - void Awake() - { - WebCamDevice device = WebCamTexture.devices[camdex]; - tex = new WebCamTexture(device.name); - display.texture = tex; - - tex.Play(); - } - - /// - /// Function to toggle between stopping and starting - /// - /* - public void toggle() - { - if (tex.isPlaying) - { - tex.Stop(); - } - else - { - tex.Play(); - } - } - */ - public void PlayCam() - { - if (!tex.isPlaying) tex.Play(); - } - - public void StopCam() - { - if (tex.isPlaying) tex.Stop(); - } - - /// - /// Swap webcam by cycling through the `WebCamTexture.devices` list - /// - public void SwapCam() - { - if (WebCamTexture.devices.Length > 0) - { - // Stop the old camera - display.texture = null; - tex.Stop(); - tex = null; - - // Find the new camera - camdex += 1; - camdex %= WebCamTexture.devices.Length; - - // Start the new camera - WebCamDevice device = WebCamTexture.devices[camdex]; - tex = new WebCamTexture(device.name); - display.texture = tex; - - tex.Play(); - } - } - - /// - /// Scene changing is implemented here to avoid problems with webcam - /// - public void GotoThemeSelection() - { - display.texture = null; - tex.Stop(); - tex = null; - - SystemController.GetInstance().BackToPreviousScene(); - } -} diff --git a/Assets/Hangman/Scripts/WebCam.cs.meta b/Assets/Hangman/Scripts/WebCam.cs.meta deleted file mode 100644 index 37b2cee..0000000 --- a/Assets/Hangman/Scripts/WebCam.cs.meta +++ /dev/null @@ -1,11 +0,0 @@ -fileFormatVersion: 2 -guid: 5db51e2552e03de4b9e7e91b5746adbc -MonoImporter: - externalObjects: {} - serializedVersion: 2 - defaultReferences: [] - executionOrder: 0 - icon: {instanceID: 0} - userData: - assetBundleName: - assetBundleVariant: diff --git a/Assets/MediaPipeUnity/Interfaces.meta b/Assets/MediaPipeUnity/Interfaces.meta new file mode 100644 index 0000000..80634de --- /dev/null +++ b/Assets/MediaPipeUnity/Interfaces.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: 73c615986873dc246893879daf74c05d +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Assets/MediaPipeUnity/Interfaces/Listener.cs b/Assets/MediaPipeUnity/Interfaces/Listener.cs new file mode 100644 index 0000000..05880f4 --- /dev/null +++ b/Assets/MediaPipeUnity/Interfaces/Listener.cs @@ -0,0 +1,14 @@ +using System.Collections; +using System.Collections.Generic; +using UnityEngine; +/// +/// Listener interface with an IEnumerator as its processing-function. +/// +public interface Listener +{ + /// + /// The function that is called by the publisher. + /// + /// + public IEnumerator ProcessIncomingCall(); +} diff --git a/Assets/MediaPipeUnity/Scripts/Feedback.cs.meta b/Assets/MediaPipeUnity/Interfaces/Listener.cs.meta similarity index 83% rename from Assets/MediaPipeUnity/Scripts/Feedback.cs.meta rename to Assets/MediaPipeUnity/Interfaces/Listener.cs.meta index cd8d001..704b3ba 100644 --- a/Assets/MediaPipeUnity/Scripts/Feedback.cs.meta +++ b/Assets/MediaPipeUnity/Interfaces/Listener.cs.meta @@ -1,5 +1,5 @@ fileFormatVersion: 2 -guid: 44e682a32ee15cc489bf50f3a06f717b +guid: e4c1da9896d9ba2449549a016b5fd15e MonoImporter: externalObjects: {} serializedVersion: 2 diff --git a/Assets/MediaPipeUnity/Interfaces/SignPredictorInterfaces.asmdef b/Assets/MediaPipeUnity/Interfaces/SignPredictorInterfaces.asmdef new file mode 100644 index 0000000..a39877a --- /dev/null +++ b/Assets/MediaPipeUnity/Interfaces/SignPredictorInterfaces.asmdef @@ -0,0 +1,14 @@ +{ + "name": "SignPredictorInterfaces", + "rootNamespace": "", + "references": [], + "includePlatforms": [], + "excludePlatforms": [], + "allowUnsafeCode": false, + "overrideReferences": false, + "precompiledReferences": [], + "autoReferenced": true, + "defineConstraints": [], + "versionDefines": [], + "noEngineReferences": false +} \ No newline at end of file diff --git a/Assets/MediaPipeUnity/Interfaces/SignPredictorInterfaces.asmdef.meta b/Assets/MediaPipeUnity/Interfaces/SignPredictorInterfaces.asmdef.meta new file mode 100644 index 0000000..c87212f --- /dev/null +++ b/Assets/MediaPipeUnity/Interfaces/SignPredictorInterfaces.asmdef.meta @@ -0,0 +1,7 @@ +fileFormatVersion: 2 +guid: f55a02e98b01bc849b30d9650ccd8f15 +AssemblyDefinitionImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Assets/MediaPipeUnity/Prefabs/Feedback.prefab b/Assets/MediaPipeUnity/Prefabs/Feedback.prefab index ee82a36..47e40cf 100644 --- a/Assets/MediaPipeUnity/Prefabs/Feedback.prefab +++ b/Assets/MediaPipeUnity/Prefabs/Feedback.prefab @@ -9,7 +9,6 @@ GameObject: serializedVersion: 6 m_Component: - component: {fileID: 4318122119930585316} - - component: {fileID: 4318122119930585317} m_Layer: 5 m_Name: Feedback m_TagString: Untagged @@ -39,22 +38,6 @@ RectTransform: m_AnchoredPosition: {x: 0, y: 200} m_SizeDelta: {x: 500, y: 150} m_Pivot: {x: 0.5, y: 0} ---- !u!114 &4318122119930585317 -MonoBehaviour: - m_ObjectHideFlags: 0 - m_CorrespondingSourceObject: {fileID: 0} - m_PrefabInstance: {fileID: 0} - m_PrefabAsset: {fileID: 0} - m_GameObject: {fileID: 4318122119930585319} - m_Enabled: 1 - m_EditorHideFlags: 0 - m_Script: {fileID: 11500000, guid: 44e682a32ee15cc489bf50f3a06f717b, type: 3} - m_Name: - m_EditorClassIdentifier: - feedbackText: {fileID: 4318122120222767928} - feedbackProgress: {fileID: 4318122119968934242} - feedbackProgressImage: {fileID: 4318122120334233319} - signPredictor: {fileID: 0} --- !u!1 &4318122119968934244 GameObject: m_ObjectHideFlags: 0 diff --git a/Assets/MediaPipeUnity/ScriptableObjects/ModelList.asset b/Assets/MediaPipeUnity/ScriptableObjects/ModelList.asset index 12d799c..897dae7 100644 --- a/Assets/MediaPipeUnity/ScriptableObjects/ModelList.asset +++ b/Assets/MediaPipeUnity/ScriptableObjects/ModelList.asset @@ -17,4 +17,4 @@ MonoBehaviour: - index: 0 model: {fileID: 5022602860645237092, guid: e6d85df707405ad4f97c23b07227ee99, type: 3} - index: 1 - model: {fileID: 5022602860645237092, guid: e6d85df707405ad4f97c23b07227ee99, type: 3} + model: {fileID: 0} diff --git a/Assets/MediaPipeUnity/Scripts/AbstractFeedback.cs b/Assets/MediaPipeUnity/Scripts/AbstractFeedback.cs new file mode 100644 index 0000000..ab5b0c8 --- /dev/null +++ b/Assets/MediaPipeUnity/Scripts/AbstractFeedback.cs @@ -0,0 +1,40 @@ +using DigitalRuby.Tween; +using Mediapipe.Unity.Tutorial; +using System; +using System.Collections; +using TMPro; +using UnityEngine; +using UnityEngine.Events; +using UnityEngine.UI; + +/// +/// Class to display feedback during a course +/// +public abstract class AbstractFeedback : MonoBehaviour, Listener +{ + + /// + /// Reference to the sign predictor + /// + public SignPredictor signPredictor; + /// + /// The function that is called by the publisher on all its listeners + /// + /// + public IEnumerator ProcessIncomingCall() + { + yield return StartCoroutine(UpdateFeedback()); + } + /// + /// A function to add yourself as listener to the signPredictor you are holding + /// + public void AddSelfAsListener() + { + signPredictor.listeners.Add(this); + } + /// + /// The function that holds the logic to process the new probabilities of the signPredictor + /// + /// + protected abstract IEnumerator UpdateFeedback(); +} \ No newline at end of file diff --git a/Assets/Hangman/Scripts/HangManWebcam.cs.meta b/Assets/MediaPipeUnity/Scripts/AbstractFeedback.cs.meta similarity index 83% rename from Assets/Hangman/Scripts/HangManWebcam.cs.meta rename to Assets/MediaPipeUnity/Scripts/AbstractFeedback.cs.meta index de815e7..5fdbcf9 100644 --- a/Assets/Hangman/Scripts/HangManWebcam.cs.meta +++ b/Assets/MediaPipeUnity/Scripts/AbstractFeedback.cs.meta @@ -1,5 +1,5 @@ fileFormatVersion: 2 -guid: a62d2b8bded916443835dc19010b83c1 +guid: 7b5ac794337a54143a6e3077483d96c9 MonoImporter: externalObjects: {} serializedVersion: 2 diff --git a/Assets/MediaPipeUnity/Scripts/Feedback.cs b/Assets/MediaPipeUnity/Scripts/Feedback.cs deleted file mode 100644 index bbb88a0..0000000 --- a/Assets/MediaPipeUnity/Scripts/Feedback.cs +++ /dev/null @@ -1,182 +0,0 @@ -using DigitalRuby.Tween; -using Mediapipe.Unity.Tutorial; -using System; -using System.Collections; -using TMPro; -using UnityEngine; -using UnityEngine.Events; -using UnityEngine.UI; - -/// -/// Class to display feedback during a course -/// -public class Feedback : MonoBehaviour -{ - /// - /// Reference to the feedback field - /// - public TMP_Text feedbackText; - - /// - /// Reference to the progress bar - /// - public Slider feedbackProgress; - - /// - /// Reference to the progress bar image, so we can add fancy colors - /// - public Image feedbackProgressImage; - - /// - /// Reference to the sign predictor - /// - public SignPredictor signPredictor; - - /// - /// Callback for getting the correct sign - /// - public Func getSignCallback; - - /// - /// Callback to initiate the next sign - /// - public UnityAction predictSignCallback; - - /// - /// Timer to keep track of how long a incorrect sign is performed - /// - private DateTime timer; - - /// - /// Current predicted sign - /// - private string predictedSign = null; - - /// - /// Previous incorrect sign, so we can keep track whether the user is wrong or the user is still changing signs - /// - private string previousIncorrectSign = null; - - /// - /// Start is called before the first frame update - /// - void Start() - { - // Start the coroutine to update the scale every 200 milliseconds - StartCoroutine(UpdateFeedback()); - } - - /// - /// UpdateScale updates the progress bar every 200ms, updated the feedback text, and progress bar color - /// If a high enough accuracy is detected, it will go to the next sign - /// - /// - IEnumerator UpdateFeedback() - { - while (true) - { - if (getSignCallback != null && predictSignCallback != null) - { - - // Get current sign - string currentSign = getSignCallback(); - // Get the predicted sign - if (signPredictor != null && signPredictor.learnableProbabilities != null && - currentSign != null && signPredictor.learnableProbabilities.ContainsKey(currentSign)) - { - float accuracy = signPredictor.learnableProbabilities[currentSign]; - if (feedbackText != null && feedbackProgressImage != null){ - if (accuracy > 0.98) - { - // TODO: fix emojis - feedbackText.text = "✨ Perfect ✨"; - Color col = new Color(0xff / 255.0f, 0xcc / 255.0f, 0x00 / 255.0f); - feedbackText.color = col; - feedbackProgressImage.color = col; - } - else if (accuracy > 0.95) - { - feedbackText.text = "Super!"; - Color col = new Color(0x00 / 255.0f, 0xff / 255.0f, 0xcc / 255.0f); - feedbackText.color = col; - feedbackProgressImage.color = col; - } - else if (accuracy > 0.90) - { - feedbackText.text = "Goed"; - feedbackText.color = Color.green; - feedbackProgressImage.color = Color.green; - } - else if (accuracy > 0.80) - { - feedbackText.text = "Bijna..."; - Color col = new Color(0xff / 255.0f, 0x66 / 255.0f, 0x00 / 255.0f); - feedbackText.color = col; - feedbackProgressImage.color = col; - } - else - { - feedbackText.text = "Detecteren..."; - feedbackText.color = Color.red; - feedbackProgressImage.color = Color.red; - } - - float oldValue = feedbackProgress.value; - // use an exponential scale - float newValue = Mathf.Exp(4 * (accuracy - 1.0f)); - feedbackProgress.gameObject.Tween("FeedbackUpdate", oldValue, newValue, 0.2f, TweenScaleFunctions.CubicEaseInOut, (t) => - { - if (feedbackProgress != null) - { - feedbackProgress.value = t.CurrentValue; - } - }); - } - - // Check whether (in)correct sign has high accuracy - foreach (var kv in signPredictor.learnableProbabilities) - { - if (kv.Value > 0.90) - { - predictedSign = kv.Key; - // Correct sign - if (predictedSign == currentSign) - { - yield return new WaitForSeconds(1.0f); - predictSignCallback(predictedSign); - timer = DateTime.Now; - predictedSign = null; - previousIncorrectSign = null; - } - // Incorrect sign - else - { - if (previousIncorrectSign != predictedSign) - { - timer = DateTime.Now; - previousIncorrectSign = predictedSign; - } - else if (DateTime.Now - timer > TimeSpan.FromSeconds(2.0f)) - { - predictSignCallback(predictedSign); - timer = DateTime.Now; - predictedSign = null; - previousIncorrectSign = null; - } - } - break; - } - } - } - else if(feedbackProgress != null) - { - - feedbackProgress.value = 0.0f; - } - } - - // Wait for 200 milliseconds before updating the scale again - yield return new WaitForSeconds(0.2f); - } - } -} \ No newline at end of file diff --git a/Assets/MediaPipeUnity/Scripts/SignPredictor.asmdef b/Assets/MediaPipeUnity/Scripts/SignPredictor.asmdef index ba94856..2a113bb 100644 --- a/Assets/MediaPipeUnity/Scripts/SignPredictor.asmdef +++ b/Assets/MediaPipeUnity/Scripts/SignPredictor.asmdef @@ -7,7 +7,8 @@ "GUID:04c4d86a70aa56c55a78c61f1ab1a56d", "GUID:edc93f477bb73a743a97d6882ed330b3", "GUID:58e104b97fb3752438ada2902a36dcbf", - "GUID:7f2d0ee6dd21e1d4eb25b71b7a749d25" + "GUID:7f2d0ee6dd21e1d4eb25b71b7a749d25", + "GUID:f55a02e98b01bc849b30d9650ccd8f15" ], "includePlatforms": [], "excludePlatforms": [], diff --git a/Assets/MediaPipeUnity/Scripts/SignPredictor.cs b/Assets/MediaPipeUnity/Scripts/SignPredictor.cs index 2ec4263..4648e26 100644 --- a/Assets/MediaPipeUnity/Scripts/SignPredictor.cs +++ b/Assets/MediaPipeUnity/Scripts/SignPredictor.cs @@ -141,6 +141,8 @@ namespace Mediapipe.Unity.Tutorial /// private Tensor inputTensor; + public List listeners = new List(); + /// /// Google Mediapipe setup & run /// @@ -159,6 +161,7 @@ namespace Mediapipe.Unity.Tutorial webcamTexture.Play(); + yield return new WaitUntil(() => webcamTexture.width > 16); // Set webcam aspect ratio @@ -167,63 +170,63 @@ namespace Mediapipe.Unity.Tutorial float webcamAspect = (float)webcamTexture.width / (float)webcamTexture.height; screen.rectTransform.sizeDelta = new Vector2(screen.rectTransform.sizeDelta.y * webcamAspect, (screen.rectTransform.sizeDelta.y)); screen.texture = webcamTexture; - if(screen2 != null) + if (screen2 != null) { screen2.rectTransform.sizeDelta = new Vector2(screen2.rectTransform.sizeDelta.y * webcamAspect, (screen2.rectTransform.sizeDelta.y)); } - // TODO this method is kinda meh you should use - inputTexture = new Texture2D(width, height, TextureFormat.RGBA32, false); - pixelData = new Color32[width * height]; - - if (!resourceManagerIsInitialized) + if (modelList.GetCurrentModel() != null) { - resourceManager = new StreamingAssetsResourceManager(); - yield return resourceManager.PrepareAssetAsync("pose_detection.bytes"); - yield return resourceManager.PrepareAssetAsync("pose_landmark_full.bytes"); - yield return resourceManager.PrepareAssetAsync("face_landmark.bytes"); - yield return resourceManager.PrepareAssetAsync("hand_landmark_full.bytes"); - yield return resourceManager.PrepareAssetAsync("face_detection_short_range.bytes"); - yield return resourceManager.PrepareAssetAsync("hand_recrop.bytes"); - yield return resourceManager.PrepareAssetAsync("handedness.txt"); - resourceManagerIsInitialized = true; + // TODO this method is kinda meh you should use + inputTexture = new Texture2D(width, height, TextureFormat.RGBA32, false); + pixelData = new Color32[width * height]; + + if (!resourceManagerIsInitialized) + { + resourceManager = new StreamingAssetsResourceManager(); + yield return resourceManager.PrepareAssetAsync("pose_detection.bytes"); + yield return resourceManager.PrepareAssetAsync("pose_landmark_full.bytes"); + yield return resourceManager.PrepareAssetAsync("face_landmark.bytes"); + yield return resourceManager.PrepareAssetAsync("hand_landmark_full.bytes"); + yield return resourceManager.PrepareAssetAsync("face_detection_short_range.bytes"); + yield return resourceManager.PrepareAssetAsync("hand_recrop.bytes"); + yield return resourceManager.PrepareAssetAsync("handedness.txt"); + resourceManagerIsInitialized = true; + } + + stopwatch = new Stopwatch(); + + // Setting up the graph + graph = new CalculatorGraph(configAsset.text); + + posestream = new OutputStream(graph, "pose_landmarks", "pose_landmarks_presence"); + leftstream = new OutputStream(graph, "left_hand_landmarks", "left_hand_landmarks_presence"); + rightstream = new OutputStream(graph, "right_hand_landmarks", "right_hand_landmarks_presence"); + + posestream.StartPolling().AssertOk(); + leftstream.StartPolling().AssertOk(); + rightstream.StartPolling().AssertOk(); + + graph.StartRun().AssertOk(); + stopwatch.Start(); + + + keypointManager = new KeypointManager(modelInfoFile); + // check if model exists at path + //var model = ModelLoader.Load(Resources.Load("Models/Fingerspelling/model_A-L")); + worker = modelList.GetCurrentModel().CreateWorker(); + + StartCoroutine(SignRecognitionCoroutine()); + StartCoroutine(MediapipeCoroutine()); } - - stopwatch = new Stopwatch(); - - // Setting up the graph - graph = new CalculatorGraph(configAsset.text); - - posestream = new OutputStream(graph, "pose_landmarks", "pose_landmarks_presence"); - leftstream = new OutputStream(graph, "left_hand_landmarks", "left_hand_landmarks_presence"); - rightstream = new OutputStream(graph, "right_hand_landmarks", "right_hand_landmarks_presence"); - - posestream.StartPolling().AssertOk(); - leftstream.StartPolling().AssertOk(); - rightstream.StartPolling().AssertOk(); - - graph.StartRun().AssertOk(); - stopwatch.Start(); - - - keypointManager = new KeypointManager(modelInfoFile); - - // check if model exists at path - //var model = ModelLoader.Load(Resources.Load("Models/Fingerspelling/model_A-L")); - worker = modelList.GetCurrentModel().CreateWorker(); - - StartCoroutine(SignRecognitionCoroutine()); - StartCoroutine(MediapipeCoroutine()); } - - public void ChangeModel(ModelIndex index) + /// + /// Called at the start of course/Minigame, will set the model before the start of SIgnPredictor is called. + /// + /// The index of the model to be used + public void SetModel(ModelIndex index) { this.modelList.SetCurrentModel(index); - // If a worker already existed, we throw it out - worker?.Dispose(); - - // Add a new worker for the new model - worker = modelList.GetCurrentModel().CreateWorker(); } /// @@ -325,6 +328,10 @@ namespace Mediapipe.Unity.Tutorial learnableProbabilities.Add(((char)(i + 65)).ToString(), softmaxedOutput2[i]); } //Debug.Log($"prob = [{learnableProbabilities.Aggregate(" ", (t, kv) => $"{t}{kv.Key}:{kv.Value} ")}]"); + foreach(Listener listener in listeners) + { + yield return listener.ProcessIncomingCall(); + } } else { diff --git a/Assets/SpellingBee/PlayModeTests/SpellingBeePlayModeTests.asmdef b/Assets/SpellingBee/PlayModeTests/SpellingBeePlayModeTests.asmdef index a332ab7..3cb15ac 100644 --- a/Assets/SpellingBee/PlayModeTests/SpellingBeePlayModeTests.asmdef +++ b/Assets/SpellingBee/PlayModeTests/SpellingBeePlayModeTests.asmdef @@ -7,7 +7,8 @@ "InterfacesScripts", "Unity.TextMeshPro", "SpellingBeeScripts", - "AccountsScripts" + "AccountsScripts", + "SignPredictor" ], "includePlatforms": [], "excludePlatforms": [], diff --git a/Assets/SpellingBee/Scenes/SpellingBeeGame.unity b/Assets/SpellingBee/Scenes/SpellingBeeGame.unity index 64d9253..ae5d17e 100644 --- a/Assets/SpellingBee/Scenes/SpellingBeeGame.unity +++ b/Assets/SpellingBee/Scenes/SpellingBeeGame.unity @@ -1741,13 +1741,35 @@ RectTransform: m_PrefabAsset: {fileID: 0} --- !u!114 &967164045 stripped MonoBehaviour: - m_CorrespondingSourceObject: {fileID: 4318122119930585317, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3} + m_CorrespondingSourceObject: {fileID: 4318122120334233319, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3} m_PrefabInstance: {fileID: 967164043} m_PrefabAsset: {fileID: 0} m_GameObject: {fileID: 0} m_Enabled: 1 m_EditorHideFlags: 0 - m_Script: {fileID: 11500000, guid: 44e682a32ee15cc489bf50f3a06f717b, type: 3} + m_Script: {fileID: 11500000, guid: fe87c0e1cc204ed48ad3b37840f39efc, type: 3} + m_Name: + m_EditorClassIdentifier: +--- !u!114 &967164046 stripped +MonoBehaviour: + m_CorrespondingSourceObject: {fileID: 4318122119968934242, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3} + m_PrefabInstance: {fileID: 967164043} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 0} + m_Enabled: 1 + m_EditorHideFlags: 0 + m_Script: {fileID: 11500000, guid: 67db9e8f0e2ae9c40bc1e2b64352a6b4, type: 3} + m_Name: + m_EditorClassIdentifier: +--- !u!114 &967164047 stripped +MonoBehaviour: + m_CorrespondingSourceObject: {fileID: 4318122120222767928, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3} + m_PrefabInstance: {fileID: 967164043} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 0} + m_Enabled: 1 + m_EditorHideFlags: 0 + m_Script: {fileID: 11500000, guid: f4688fdb7df04437aeb418b961361dc5, type: 3} m_Name: m_EditorClassIdentifier: --- !u!1 &978093274 @@ -3584,6 +3606,7 @@ MonoBehaviour: m_Script: {fileID: 11500000, guid: 44fbed5ae228de39b9f727def7578d06, type: 3} m_Name: m_EditorClassIdentifier: + signPredictor: {fileID: 1592592444} themeList: {fileID: 11400000, guid: a247e2ce790f0f746a3bc521e6ab7d58, type: 2} userList: {fileID: 11400000, guid: 072bec636a40f7e4e93b0ac624a3bda2, type: 2} minigame: {fileID: 11400000, guid: 8a087d241d652634eb4f6352267ea7dc, type: 2} @@ -3594,8 +3617,10 @@ MonoBehaviour: timerText: {fileID: 1843239267} bonusTimeText: {fileID: 1812475780} Scoreboard: {fileID: 862382568} - feedback: {fileID: 967164045} gameEndedPanel: {fileID: 757133117} + feedbackText: {fileID: 967164047} + feedbackProgress: {fileID: 967164046} + feedbackProgressImage: {fileID: 967164045} --- !u!1 &1499197558 GameObject: m_ObjectHideFlags: 0 @@ -3957,7 +3982,6 @@ MonoBehaviour: m_Name: m_EditorClassIdentifier: modelList: {fileID: 11400000, guid: 39516e4e6e56f0f4f80647d9c4d8034c, type: 2} - model: {fileID: 5022602860645237092, guid: e6d85df707405ad4f97c23b07227ee99, type: 3} modelInfoFile: {fileID: 4900000, guid: fb8b51022bdcd654a9f29c054832a1b5, type: 3} configAsset: {fileID: 4900000, guid: 6288c43cdca97374782dac1ea87aa029, type: 3} screen: {fileID: 1743003084} diff --git a/Assets/SpellingBee/Scripts/SpellingBeeController.cs b/Assets/SpellingBee/Scripts/SpellingBeeController.cs index 0da99a5..71c5fd0 100644 --- a/Assets/SpellingBee/Scripts/SpellingBeeController.cs +++ b/Assets/SpellingBee/Scripts/SpellingBeeController.cs @@ -5,8 +5,9 @@ using System.Linq; using TMPro; using UnityEngine; using UnityEngine.UI; +using DigitalRuby.Tween; -public partial class SpellingBeeController : MonoBehaviour +public partial class SpellingBeeController : AbstractFeedback { /// /// All of the words that can be used in this session @@ -136,20 +137,55 @@ public partial class SpellingBeeController : MonoBehaviour /// public Transform Scoreboard; - /// - /// Accuracy feeback object - /// - public Feedback feedback; - /// /// Reference to the gameEnded panel, so we can update its display /// public GameObject gameEndedPanel; + /// + /// Reference to the feedback field + /// + public TMP_Text feedbackText; + + /// + /// Reference to the progress bar + /// + public Slider feedbackProgress; + + /// + /// Reference to the progress bar image, so we can add fancy colors + /// + public Image feedbackProgressImage; + + /// + /// Timer to keep track of how long a incorrect sign is performed + /// + protected DateTime timer; + + /// + /// Current predicted sign + /// + protected string predictedSign = null; + + /// + /// Previous incorrect sign, so we can keep track whether the user is wrong or the user is still changing signs + /// + protected string previousIncorrectSign = null; + /// /// Start is called before the first frame update /// public void Start() + { + StartController(); + + signPredictor.SetModel(currentTheme.modelIndex); + AddSelfAsListener(); + } + /// + /// Is called at the start of the scene AND when the game is replayed + /// + public void StartController() { correctLetters = 0; incorrectLetters = 0; @@ -182,29 +218,10 @@ public partial class SpellingBeeController : MonoBehaviour userList.Save(); currentTheme = minigame.themeList.themes[minigame.themeList.currentThemeIndex]; - feedback.signPredictor.ChangeModel(currentTheme.modelIndex); + //feedback.signPredictor.ChangeModel(currentTheme.modelIndex); words.AddRange(currentTheme.learnables); ShuffleWords(); NextWord(); - - // Set calllbacks - feedback.getSignCallback = () => - { - if (letterIndex < currentWord.Length) - { - return currentWord[letterIndex].ToString().ToUpper(); - } - return null; - }; - feedback.predictSignCallback = (sign) => - { - bool successful = sign.ToUpper() == currentWord[letterIndex].ToString().ToUpper(); - if (successful) - { - AddSeconds(secondsPerLetter); - } - NextLetter(successful); - }; } /// @@ -453,4 +470,116 @@ public partial class SpellingBeeController : MonoBehaviour { yield return new WaitForSecondsRealtime(2); } + + /// + /// The updateFunction that is called when new probabilities become available + /// + /// + protected override IEnumerator UpdateFeedback() + { + // Get current sign + string currentSign = GetSign(); + // Get the predicted sign + if (signPredictor != null && signPredictor.learnableProbabilities != null && + currentSign != null && signPredictor.learnableProbabilities.ContainsKey(currentSign)) + { + float accuracy = signPredictor.learnableProbabilities[currentSign]; + if (feedbackText != null && feedbackProgressImage != null) + { + if (accuracy > 0.90) + { + feedbackText.text = "Goed"; + feedbackText.color = Color.green; + feedbackProgressImage.color = Color.green; + } + else if (accuracy > 0.80) + { + feedbackText.text = "Bijna..."; + Color col = new Color(0xff / 255.0f, 0x66 / 255.0f, 0x00 / 255.0f); + feedbackText.color = col; + feedbackProgressImage.color = col; + } + else + { + feedbackText.text = "Detecteren..."; + feedbackText.color = Color.red; + feedbackProgressImage.color = Color.red; + } + + float oldValue = feedbackProgress.value; + // use an exponential scale + float newValue = Mathf.Exp(4 * (accuracy - 1.0f)); + feedbackProgress.gameObject.Tween("FeedbackUpdate", oldValue, newValue, 0.2f, TweenScaleFunctions.CubicEaseInOut, (t) => + { + if (feedbackProgress != null) + { + feedbackProgress.value = t.CurrentValue; + } + }); + } + + // Check whether (in)correct sign has high accuracy + foreach (var kv in signPredictor.learnableProbabilities) + { + if (kv.Value > 0.90) + { + predictedSign = kv.Key; + // Correct sign + if (predictedSign == currentSign) + { + yield return new WaitForSeconds(1.0f); + predictSign(predictedSign); + timer = DateTime.Now; + predictedSign = null; + previousIncorrectSign = null; + } + // Incorrect sign + else + { + if (previousIncorrectSign != predictedSign) + { + timer = DateTime.Now; + previousIncorrectSign = predictedSign; + } + else if (DateTime.Now - timer > TimeSpan.FromSeconds(2.0f)) + { + predictSign(predictedSign); + timer = DateTime.Now; + predictedSign = null; + previousIncorrectSign = null; + } + } + break; + } + } + } + else if (feedbackProgress != null) + { + + feedbackProgress.value = 0.0f; + } + yield return null; + } + /// + /// Function to get the current letter that needs to be signed + /// + /// the current letter that needs to be signed + public string GetSign(){ + if (letterIndex + /// Function to confirm your prediction and check if it is correct. + /// + /// + public void predictSign(string sign) { + bool successful = sign.ToUpper() == currentWord[letterIndex].ToString().ToUpper(); + if (successful) + { + AddSeconds(secondsPerLetter); + } + NextLetter(successful); + } } diff --git a/Assets/SpellingBee/Scripts/SpellingBeeController.cs.meta b/Assets/SpellingBee/Scripts/SpellingBeeController.cs.meta index be352cc..6fa40d2 100644 --- a/Assets/SpellingBee/Scripts/SpellingBeeController.cs.meta +++ b/Assets/SpellingBee/Scripts/SpellingBeeController.cs.meta @@ -3,21 +3,7 @@ guid: 44fbed5ae228de39b9f727def7578d06 MonoImporter: externalObjects: {} serializedVersion: 2 - defaultReferences: - - input: {instanceID: 0} - - endText: {instanceID: 0} - - correctWordsText: {instanceID: 0} - - correctLettersText: {instanceID: 0} - - gameEndedPanel: {instanceID: 0} - - replayButton: {instanceID: 0} - - userList: {instanceID: 0} - - minigame: {instanceID: 0} - - letterPrefab: {instanceID: 0} - - letterContainer: {instanceID: 0} - - wordImage: {instanceID: 0} - - timerText: {instanceID: 0} - - Scoreboard: {instanceID: 0} - - scoreboardEntry: {fileID: 9154151134820372555, guid: d4a3a228b08d61847acc6da35b44e52c, type: 3} + defaultReferences: [] executionOrder: 0 icon: {instanceID: 0} userData: diff --git a/Assets/SpellingBee/Scripts/SpellingBeeScripts.asmdef b/Assets/SpellingBee/Scripts/SpellingBeeScripts.asmdef index d8c8edb..dd37fc8 100644 --- a/Assets/SpellingBee/Scripts/SpellingBeeScripts.asmdef +++ b/Assets/SpellingBee/Scripts/SpellingBeeScripts.asmdef @@ -6,8 +6,8 @@ "GUID:1631ed2680c61245b8211d943c1639a8", "GUID:3444c67d5a3a93e5a95a48906078c372", "GUID:d0b6b39a21908f94fbbd9f2c196a9725", - "GUID:5c2b5ba89f9e74e418232e154bc5cc7a", - "GUID:7f2d0ee6dd21e1d4eb25b71b7a749d25" + "GUID:7f2d0ee6dd21e1d4eb25b71b7a749d25", + "GUID:58e104b97fb3752438ada2902a36dcbf" ], "includePlatforms": [], "excludePlatforms": [],