Resolve WES-131-Feedback-REfactor

Assets/Common/PlayModeTests/CommonPlayModeTests.asmdef

@@ -8,7 +8,8 @@
         "CommonScripts",
         "InterfacesScripts",
         "Unity.TextMeshPro",
-        "AccountsScripts"
+        "AccountsScripts",
+        "SignPredictor"
     ],
     "includePlatforms": [],
     "excludePlatforms": [],

Assets/Common/Scripts/CommonScripts.asmdef

@@ -5,7 +5,6 @@
         "GUID:6055be8ebefd69e48b49212b09b47b2f",
         "GUID:63c63e721f65ebb7d871cb9ef49f4752",
         "GUID:1631ed2680c61245b8211d943c1639a8",
-        "GUID:5c2b5ba89f9e74e418232e154bc5cc7a",
         "GUID:7f2d0ee6dd21e1d4eb25b71b7a749d25"
     ],
     "includePlatforms": [],

Assets/Courses/Scenes/CourseScreen.unity

@@ -1273,7 +1273,6 @@ MonoBehaviour:
   m_Name: 
   m_EditorClassIdentifier: 
   modelList: {fileID: 11400000, guid: 39516e4e6e56f0f4f80647d9c4d8034c, type: 2}
-  model: {fileID: 5022602860645237092, guid: e6d85df707405ad4f97c23b07227ee99, type: 3}
   modelInfoFile: {fileID: 4900000, guid: fb8b51022bdcd654a9f29c054832a1b5, type: 3}
   configAsset: {fileID: 4900000, guid: 6288c43cdca97374782dac1ea87aa029, type: 3}
   screen: {fileID: 378145456}
@@ -1529,6 +1528,7 @@ MonoBehaviour:
   m_Script: {fileID: 11500000, guid: 6b3f784c065813a4a8364b1299284816, type: 3}
   m_Name: 
   m_EditorClassIdentifier: 
+  signPredictor: {fileID: 883853268}
   previewModel: {fileID: 5022602860645237092, guid: e6d85df707405ad4f97c23b07227ee99, type: 3}
   feedbackProgressBar: {fileID: 4318122121437849759}
   previewMessage: {fileID: 2070775951}
@@ -1544,7 +1544,9 @@ MonoBehaviour:
   ResultsDecription: {fileID: 100123246}
   CoursesButton: {fileID: 839294691}
   timeSpent: {fileID: 77614869}
-  feedback: {fileID: 1714882683}
+  feedbackText: {fileID: 4318122121437849762}
+  feedbackProgress: {fileID: 4318122121437849761}
+  feedbackProgressImage: {fileID: 4318122121437849760}
 --- !u!4 &1122267057
 Transform:
   m_ObjectHideFlags: 0
@@ -2284,17 +2286,6 @@ RectTransform:
   m_CorrespondingSourceObject: {fileID: 4318122119930585316, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3}
   m_PrefabInstance: {fileID: 4318122121437849758}
   m_PrefabAsset: {fileID: 0}
---- !u!114 &1714882683 stripped
-MonoBehaviour:
-  m_CorrespondingSourceObject: {fileID: 4318122119930585317, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3}
-  m_PrefabInstance: {fileID: 4318122121437849758}
-  m_PrefabAsset: {fileID: 0}
-  m_GameObject: {fileID: 4318122121437849759}
-  m_Enabled: 1
-  m_EditorHideFlags: 0
-  m_Script: {fileID: 11500000, guid: 44e682a32ee15cc489bf50f3a06f717b, type: 3}
-  m_Name: 
-  m_EditorClassIdentifier: 
 --- !u!1 &1773033262
 GameObject:
   m_ObjectHideFlags: 0
@@ -2897,3 +2888,36 @@ GameObject:
   m_CorrespondingSourceObject: {fileID: 4318122119930585319, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3}
   m_PrefabInstance: {fileID: 4318122121437849758}
   m_PrefabAsset: {fileID: 0}
+--- !u!114 &4318122121437849760 stripped
+MonoBehaviour:
+  m_CorrespondingSourceObject: {fileID: 4318122120334233319, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3}
+  m_PrefabInstance: {fileID: 4318122121437849758}
+  m_PrefabAsset: {fileID: 0}
+  m_GameObject: {fileID: 0}
+  m_Enabled: 1
+  m_EditorHideFlags: 0
+  m_Script: {fileID: 11500000, guid: fe87c0e1cc204ed48ad3b37840f39efc, type: 3}
+  m_Name: 
+  m_EditorClassIdentifier: 
+--- !u!114 &4318122121437849761 stripped
+MonoBehaviour:
+  m_CorrespondingSourceObject: {fileID: 4318122119968934242, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3}
+  m_PrefabInstance: {fileID: 4318122121437849758}
+  m_PrefabAsset: {fileID: 0}
+  m_GameObject: {fileID: 0}
+  m_Enabled: 1
+  m_EditorHideFlags: 0
+  m_Script: {fileID: 11500000, guid: 67db9e8f0e2ae9c40bc1e2b64352a6b4, type: 3}
+  m_Name: 
+  m_EditorClassIdentifier: 
+--- !u!114 &4318122121437849762 stripped
+MonoBehaviour:
+  m_CorrespondingSourceObject: {fileID: 4318122120222767928, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3}
+  m_PrefabInstance: {fileID: 4318122121437849758}
+  m_PrefabAsset: {fileID: 0}
+  m_GameObject: {fileID: 0}
+  m_Enabled: 1
+  m_EditorHideFlags: 0
+  m_Script: {fileID: 11500000, guid: f4688fdb7df04437aeb418b961361dc5, type: 3}
+  m_Name: 
+  m_EditorClassIdentifier: 

Assets/Courses/Scripts/CourseScripts.asmdef

@@ -5,8 +5,8 @@
         "Unity.TextMeshPro",
         "AccountsScripts",
         "InterfacesScripts",
-        "SignPredictor",
+        "Tween",
-        "Unity.Barracuda"
+        "SignPredictor"
     ],
     "includePlatforms": [],
     "excludePlatforms": [],

Assets/Courses/Scripts/CoursesController.cs

@@ -1,6 +1,7 @@
+using DigitalRuby.Tween;
 using System;
+using System.Collections;
 using TMPro;
-using Unity.Barracuda;
 using UnityEngine;
 using UnityEngine.UI;
 using UnityEngine.Video;
@@ -8,13 +9,10 @@ using UnityEngine.Video;
 /// <summary>
 /// TemplateCourse scene manager
 /// </summary>
-public class CoursesController : MonoBehaviour
+public class CoursesController : AbstractFeedback
 {
-    // vvv TEMPORARY STUFF vvv
-    public NNModel previewModel;
     public GameObject feedbackProgressBar;
     public GameObject previewMessage;
-    // ^^^ TEMPORARY STUFF ^^^
 
     /// <summary>
     /// Reference to instructional video player
@@ -112,10 +110,37 @@ public class CoursesController : MonoBehaviour
     /// </summary>
     public TMP_Text timeSpent;
 
+
     /// <summary>
-    /// Reference to the feedback script on the Feedback prefab
+    /// Reference to the feedback field
     /// </summary>
-    public Feedback feedback;
+    public TMP_Text feedbackText;
+
+    /// <summary>
+    /// Reference to the progress bar
+    /// </summary>
+    public Slider feedbackProgress;
+
+    /// <summary>
+    /// Reference to the progress bar image, so we can add fancy colors
+    /// </summary>
+    public Image feedbackProgressImage;
+
+    /// <summary>
+    /// Timer to keep track of how long a incorrect sign is performed
+    /// </summary>
+    protected DateTime timer;
+
+    /// <summary>
+    /// Current predicted sign
+    /// </summary>
+    protected string predictedSign = null;
+
+    /// <summary>
+    /// Previous incorrect sign, so we can keep track whether the user is wrong or the user is still changing signs
+    /// </summary>
+    protected string previousIncorrectSign = null;
+
 
     /// <summary>
     /// This function is called when the script is initialised.
@@ -124,19 +149,25 @@ public class CoursesController : MonoBehaviour
     /// Then it checks whether or not the User has started the course yet, to possibly create a new progress atribute for the course.
     /// Then it sets up the course-screen to display relevant information from the course-scriptable.
     /// </summary>
-    void Awake()
+    void Start()
+    {
+        StartCourseController();
+
+        signPredictor.SetModel(course.theme.modelIndex);
+        AddSelfAsListener();
+    }
+    /// <summary>
+    /// Holds the course-specific logic to start the controller, it is seperated to allow the course to be reset (if that would become needed)
+    /// </summary>
+    public void StartCourseController()
     {
         // Setting up course
         course = courselist.courses[courselist.currentCourseIndex];
-        feedback.signPredictor.ChangeModel(course.theme.modelIndex);
         maxWords = course.theme.learnables.Count;
 
-        // vvv TEMPORARY STUFF vvv
+        // Show preview messages if there is no model
         feedbackProgressBar.SetActive(course.theme.modelIndex != ModelIndex.NONE);
         previewMessage.SetActive(course.theme.modelIndex == ModelIndex.NONE);
-        // Instead, the NONE-modelIndex points to Fingerspelling, which gives the same result
-        //feedback.signPredictor.model = previewModel;
-        // ^^^ TEMPORARY STUFF ^^^
 
         // Create entry in current user for keeping track of progress
         userList.Load();
@@ -164,23 +195,6 @@ public class CoursesController : MonoBehaviour
         ResultPanel.SetActive(false);
         // Set the startTime
         startMoment = DateTime.Now;
-
-        // Set callbacks
-        feedback.getSignCallback = () =>
-        {
-            if (currentWordIndex < course.theme.learnables.Count)
-            {
-                return course.theme.learnables[currentWordIndex].name;
-            }
-            return null;
-        };
-        feedback.predictSignCallback = (sign) =>
-        {
-            if (sign == course.theme.learnables[currentWordIndex].name)
-            {
-                NextSign();
-            }
-        };
     }
 
     /// <summary>
@@ -287,4 +301,104 @@ public class CoursesController : MonoBehaviour
         progress.AddOrUpdate<float>("courseProgress", 1f);
         userList.Save();
     }
+    /// <summary>
+    /// The updateFunction that is called when new probabilities become available
+    /// </summary>
+    /// <returns></returns>
+    protected override IEnumerator UpdateFeedback()
+    {
+        // Get current sign
+        string currentSign = course.theme.learnables[currentWordIndex].name;
+        // Get the predicted sign
+        if (signPredictor != null && signPredictor.learnableProbabilities != null &&
+            currentSign != null && signPredictor.learnableProbabilities.ContainsKey(currentSign))
+        {
+            float accuracy = signPredictor.learnableProbabilities[currentSign];
+            if (feedbackText != null && feedbackProgressImage != null)
+            {
+                if (accuracy > 0.90)
+                {
+                    feedbackText.text = "Goed";
+                    feedbackText.color = Color.green;
+                    feedbackProgressImage.color = Color.green;
+                }
+                else if (accuracy > 0.80)
+                {
+                    feedbackText.text = "Bijna...";
+                    Color col = new Color(0xff / 255.0f, 0x66 / 255.0f, 0x00 / 255.0f);
+                    feedbackText.color = col;
+                    feedbackProgressImage.color = col;
+                }
+                else
+                {
+                    feedbackText.text = "Detecteren...";
+                    feedbackText.color = Color.red;
+                    feedbackProgressImage.color = Color.red;
+                }
+
+                float oldValue = feedbackProgress.value;
+                // use an exponential scale 
+                float newValue = Mathf.Exp(4 * (accuracy - 1.0f));
+                feedbackProgress.gameObject.Tween("FeedbackUpdate", oldValue, newValue, 0.2f, TweenScaleFunctions.CubicEaseInOut, (t) =>
+                {
+                    if (feedbackProgress != null)
+                    {
+                        feedbackProgress.value = t.CurrentValue;
+                    }
+                });
+            }
+
+            // Check whether (in)correct sign has high accuracy
+            foreach (var kv in signPredictor.learnableProbabilities)
+            {
+                if (kv.Value > 0.90)
+                {
+                    predictedSign = kv.Key;
+                    // Correct sign
+                    if (predictedSign == currentSign)
+                    {
+                        yield return new WaitForSeconds(1.0f);
+                        CheckEquality(predictedSign);
+                        timer = DateTime.Now;
+                        predictedSign = null;
+                        previousIncorrectSign = null;
+                    }
+                    // Incorrect sign
+                    else
+                    {
+                        if (previousIncorrectSign != predictedSign)
+                        {
+                            timer = DateTime.Now;
+                            previousIncorrectSign = predictedSign;
+                        }
+                        else if (DateTime.Now - timer > TimeSpan.FromSeconds(2.0f))
+                        {
+                            CheckEquality(predictedSign);
+                            timer = DateTime.Now;
+                            predictedSign = null;
+                            previousIncorrectSign = null;
+                        }
+                    }
+                    break;
+                }
+            }
+        }
+        else if (feedbackProgress != null)
+        {
+
+            feedbackProgress.value = 0.0f;
+        }
+
+    }
+    /// <summary>
+    /// Function to check equality between the current sign and the sign that the model predicted, if they are equal then the next sign is fetched.
+    /// </summary>
+    /// <param name="predicted"></param>
+    private void CheckEquality(string predicted)
+    {
+        if (predicted == course.theme.learnables[currentWordIndex].name)
+        {
+            NextSign();
+        }
+    }
 }

Assets/Hangman/Scenes/HangmanGame.unity

@@ -38,7 +38,7 @@ RenderSettings:
   m_ReflectionIntensity: 1
   m_CustomReflection: {fileID: 0}
   m_Sun: {fileID: 0}
-  m_IndirectSpecularColor: {r: 0.37311953, g: 0.38074014, b: 0.3587274, a: 1}
+  m_IndirectSpecularColor: {r: 0.37311918, g: 0.3807398, b: 0.35872716, a: 1}
   m_UseRadianceAmbientProbe: 0
 --- !u!157 &3
 LightmapSettings:
@@ -5129,6 +5129,7 @@ MonoBehaviour:
   m_Script: {fileID: 11500000, guid: 2db44635e0eb1e9429a2e6195785364d, type: 3}
   m_Name: 
   m_EditorClassIdentifier: 
+  signPredictor: {fileID: 1991376311}
   themelist: {fileID: 11400000, guid: a247e2ce790f0f746a3bc521e6ab7d58, type: 2}
   letterPrefab: {fileID: 4639383499500021565, guid: c3e66e8957864914cb022af914df6a28, type: 3}
   letterContainer: {fileID: 1870283439}
@@ -5153,8 +5154,10 @@ MonoBehaviour:
   Scoreboard: {fileID: 1007532375}
   EntriesGrid: {fileID: 1391137944}
   scoreboardEntry: {fileID: 9154151134820372555, guid: d4a3a228b08d61847acc6da35b44e52c, type: 3}
-  feedback: {fileID: 5233312447513285388}
   gottogamebutton: {fileID: 1581633295}
+  feedbackText: {fileID: 0}
+  feedbackProgress: {fileID: 0}
+  feedbackProgressImage: {fileID: 0}
 --- !u!1 &1678036720
 GameObject:
   m_ObjectHideFlags: 0
@@ -6183,7 +6186,6 @@ MonoBehaviour:
   m_Name: 
   m_EditorClassIdentifier: 
   modelList: {fileID: 11400000, guid: 39516e4e6e56f0f4f80647d9c4d8034c, type: 2}
-  model: {fileID: 5022602860645237092, guid: e6d85df707405ad4f97c23b07227ee99, type: 3}
   modelInfoFile: {fileID: 4900000, guid: fb8b51022bdcd654a9f29c054832a1b5, type: 3}
   configAsset: {fileID: 4900000, guid: 6288c43cdca97374782dac1ea87aa029, type: 3}
   screen: {fileID: 1649505745}

Assets/Hangman/Scripts/HangManWebcam.cs

@@ -1,62 +0,0 @@
-using System.Collections;
-using System.Collections.Generic;
-using UnityEngine;
-using UnityEngine.UI;
-
-public class HangManWebcam : WebCam
-{
-    /// <summary>
-    /// The display for player 1
-    /// </summary>
-    public RawImage display1;
-
-    /// <summary>
-    /// The display for player 2
-    /// </summary>
-    public RawImage display2;
-
-    /// <summary>
-    /// We use a different awake, since we dont want the camera to start immediatelly
-    /// </summary>
-    void Awake()
-    {
-        WebCamDevice device = WebCamTexture.devices[camdex];
-        tex = new WebCamTexture(device.name);
-        display.texture = tex;
-    }
-
-    /// <summary>
-    /// Hangman uses two different webcam_textures, we need to be able to toggle between them
-    /// </summary>
-    public void Switch_texture()
-    {
-        if(display == display1)
-        {
-            display = display2;
-        }
-        else 
-        {
-            display = display1; 
-        }
-        // Give the webcamTexture to the new webcam
-        display.texture = tex;
-    }
-
-    /// <summary>
-    /// Scene changing is implemented here to avoid problems with webcam
-    /// </summary>
-    public new void GotoThemeSelection()
-    {
-        //minigameList.GetIndexInMinigameList(MinigameIndex.HANGMAN);
-        if (tex != null)
-        {
-            if (tex.isPlaying)
-            {
-                display.texture = null;
-                tex.Stop();
-                tex = null;
-            }
-        }
-        SystemController.GetInstance().BackToPreviousScene();
-    }
-}

Assets/Hangman/Scripts/HangmanController.cs

@@ -1,11 +1,13 @@
 using System;
+using System.Collections;
 using System.Collections.Generic;
 using System.Linq;
 using TMPro;
 using UnityEngine;
 using UnityEngine.UI;
+using DigitalRuby.Tween;
 
-public class HangmanController : MonoBehaviour
+public class HangmanController : AbstractFeedback
 {
     /// <summary>
     /// The scriptable with all the themes, will be used to select a random word for hangman.
@@ -18,11 +20,6 @@ public class HangmanController : MonoBehaviour
     /// </summary>
     private string currentWord;
 
-    /// <summary>
-    /// All of the words that can be used in this session
-    /// </summary>
-    private string[] words;
-
     /// <summary>
     /// This integer holds the total amount of wrong guesses the player has made
     /// </summary>
@@ -200,11 +197,6 @@ public class HangmanController : MonoBehaviour
     /// </summary>
     public GameObject scoreboardEntry;
 
-    /// <summary>
-    /// Accuracy feeback object
-    /// </summary>
-    public Feedback feedback;
-
     /// <summary>
     /// The button to go into the game
     /// </summary>
@@ -215,8 +207,50 @@ public class HangmanController : MonoBehaviour
     /// </summary>
     private String currentsign = "";
 
-    // Start is called before the first frame update
+    /// <summary>
+    /// Reference to the feedback field
+    /// </summary>
+    public TMP_Text feedbackText;
+
+    /// <summary>
+    /// Reference to the progress bar
+    /// </summary>
+    public Slider feedbackProgress;
+
+    /// <summary>
+    /// Reference to the progress bar image, so we can add fancy colors
+    /// </summary>
+    public Image feedbackProgressImage;
+
+    /// <summary>
+    /// Timer to keep track of how long a incorrect sign is performed
+    /// </summary>
+    protected DateTime timer;
+
+    /// <summary>
+    /// Current predicted sign
+    /// </summary>
+    protected string predictedSign = null;
+
+    /// <summary>
+    /// Previous incorrect sign, so we can keep track whether the user is wrong or the user is still changing signs
+    /// </summary>
+    protected string previousIncorrectSign = null;
+
+    /// <summary>
+    /// Start is called before the first frame update
+    /// </summary>
     void Start()
+    {
+        StartController();
+
+        signPredictor.SetModel(ModelIndex.FINGERSPELLING);
+        AddSelfAsListener();
+    }
+    /// <summary>
+    /// Called at the start of the scene AND when the scene is replayed
+    /// </summary>
+    public void StartController()
     {
         // Make sure the mode starts at zero
         mode = 0;
@@ -240,19 +274,6 @@ public class HangmanController : MonoBehaviour
             user.minigames.Add(progress);
         }
         userList.Save();
-
-        // Hangman always uses fingerspelling
-        feedback.signPredictor.ChangeModel(ModelIndex.FINGERSPELLING);
-
-        // Set calllbacks
-        feedback.getSignCallback = () =>
-        {
-            return "A";
-        };
-        feedback.predictSignCallback = (sign) =>
-        {
-            currentsign = sign;
-        };
     }
 
     /// <summary>
@@ -281,7 +302,7 @@ public class HangmanController : MonoBehaviour
         DeleteWord();
         DisplayWord(currentWord);
 
-        replayButton.onClick.AddListener(Start);
+        replayButton.onClick.AddListener(StartController);
         // Call to display the first image, corresponding to a clean image.
         ChangeSprite();
     }
@@ -352,7 +373,7 @@ public class HangmanController : MonoBehaviour
     {
         if (mode == 1)
         {
-            if (currentsign != "")
+            if (currentsign != null && currentsign != "")
             {
                 char letter = currentsign.ToLower()[0];
                 currentsign = "";
@@ -393,7 +414,7 @@ public class HangmanController : MonoBehaviour
                 // For the first input char given by the user, check if the letter is in the word that needs to be spelled.
 
                 // Check to make sure the inputfield is not empty
-                if (currentsign != "")
+                if (currentsign != null && currentsign != "")
                 {
                     char firstLetter = currentsign.ToLower()[0];
                     currentsign = "";
@@ -543,21 +564,6 @@ public class HangmanController : MonoBehaviour
         }
     }
 
-    /// <summary>
-    /// Randomly shuffle the list of words
-    /// </summary>
-    private void ShuffleWords()
-    {
-        for (int i = words.Length - 1; i > 0; i--)
-        {
-            // Generate a random index between 0 and i (inclusive)
-            int j = UnityEngine.Random.Range(0, i + 1);
-
-            // Swap the values at indices i and j
-            (words[j], words[i]) = (words[i], words[j]);
-        }
-    }
-
     /// <summary>
     /// Update and save the scores
     /// </summary>
@@ -731,4 +737,97 @@ public class HangmanController : MonoBehaviour
             rank++;
         }
     }
+    /// <summary>
+    /// The updateFunction that is called when new probabilities become available
+    /// </summary>
+    /// <returns></returns>
+    protected override IEnumerator UpdateFeedback()
+    {
+        // Get current sign
+        string currentSign = "A";
+        // Get the predicted sign
+        if (signPredictor != null && signPredictor.learnableProbabilities != null &&
+            currentSign != null && signPredictor.learnableProbabilities.ContainsKey(currentSign))
+        {
+            float accuracy = signPredictor.learnableProbabilities[currentSign];
+            if (feedbackText != null && feedbackProgressImage != null)
+            {
+                if (accuracy > 0.90)
+                {
+                    feedbackText.text = "Goed";
+                    feedbackText.color = Color.green;
+                    feedbackProgressImage.color = Color.green;
+                }
+                else if (accuracy > 0.80)
+                {
+                    feedbackText.text = "Bijna...";
+                    Color col = new Color(0xff / 255.0f, 0x66 / 255.0f, 0x00 / 255.0f);
+                    feedbackText.color = col;
+                    feedbackProgressImage.color = col;
+                }
+                else
+                {
+                    feedbackText.text = "Detecteren...";
+                    feedbackText.color = Color.red;
+                    feedbackProgressImage.color = Color.red;
+                }
+
+                float oldValue = feedbackProgress.value;
+                // use an exponential scale 
+                float newValue = Mathf.Exp(4 * (accuracy - 1.0f));
+                feedbackProgress.gameObject.Tween("FeedbackUpdate", oldValue, newValue, 0.2f, TweenScaleFunctions.CubicEaseInOut, (t) =>
+                {
+                    if (feedbackProgress != null)
+                    {
+                        feedbackProgress.value = t.CurrentValue;
+                    }
+                });
+            }
+
+            // Check whether (in)correct sign has high accuracy
+            foreach (var kv in signPredictor.learnableProbabilities)
+            {
+                if (kv.Value > 0.90)
+                {
+                    predictedSign = kv.Key;
+                    // Correct sign
+                    if (predictedSign == currentSign)
+                    {
+                        yield return new WaitForSeconds(1.0f);
+                        CheckEquality(predictedSign);
+                        timer = DateTime.Now;
+                        predictedSign = null;
+                        previousIncorrectSign = null;
+                    }
+                    // Incorrect sign
+                    else
+                    {
+                        if (previousIncorrectSign != predictedSign)
+                        {
+                            timer = DateTime.Now;
+                            previousIncorrectSign = predictedSign;
+                        }
+                        else if (DateTime.Now - timer > TimeSpan.FromSeconds(2.0f))
+                        {
+                            CheckEquality(predictedSign);
+                            timer = DateTime.Now;
+                            predictedSign = null;
+                            previousIncorrectSign = null;
+                        }
+                    }
+                    break;
+                }
+            }
+        }
+        else if (feedbackProgress != null)
+        {
+
+            feedbackProgress.value = 0.0f;
+        }
+
+    }
+    private void CheckEquality(string sign)
+    {
+        currentsign = sign;
+    }
 }

Assets/Hangman/Scripts/WebCam.cs

@@ -1,95 +0,0 @@
-using UnityEngine.UI;
-using UnityEngine;
-
-public class WebCam : MonoBehaviour
-{
-    /// <summary>
-    /// Index of the current camera
-    /// </summary>
-    protected int camdex = 0;
-
-    /// <summary>
-    /// Texture to paste on the display
-    /// </summary>
-    protected WebCamTexture tex;
-
-    /// <summary>
-    /// Display for the video feed
-    /// </summary>
-    public RawImage display;
-
-    /// <summary>
-    /// Setup the webcam correctly
-    /// </summary>
-    void Awake()
-    {
-        WebCamDevice device = WebCamTexture.devices[camdex];
-        tex = new WebCamTexture(device.name);
-        display.texture = tex;
-
-        tex.Play();
-    }
-
-    /// <summary>
-    /// Function to toggle between stopping and starting 
-    /// </summary>
-    /*
-    public void toggle()
-    {
-        if (tex.isPlaying)
-        {
-            tex.Stop();
-        }
-        else
-        {
-            tex.Play();
-        }
-    }
-    */
-    public void PlayCam()
-    {
-        if (!tex.isPlaying) tex.Play();
-    }
-
-    public void StopCam()
-    {
-        if (tex.isPlaying) tex.Stop();
-    }
-
-    /// <summary>
-    /// Swap webcam by cycling through the `WebCamTexture.devices` list
-    /// </summary>
-    public void SwapCam()
-    {
-        if (WebCamTexture.devices.Length > 0)
-        {
-            // Stop the old camera
-            display.texture = null;
-            tex.Stop();
-            tex = null;
-
-            // Find the new camera
-            camdex += 1;
-            camdex %= WebCamTexture.devices.Length;
-
-            // Start the new camera
-            WebCamDevice device = WebCamTexture.devices[camdex];
-            tex = new WebCamTexture(device.name);
-            display.texture = tex;
-
-            tex.Play();
-        }
-    }
-
-    /// <summary>
-    /// Scene changing is implemented here to avoid problems with webcam
-    /// </summary>
-    public void GotoThemeSelection()
-    {
-        display.texture = null;
-        tex.Stop();
-        tex = null;
-
-        SystemController.GetInstance().BackToPreviousScene();
-    }
-}

Assets/Hangman/Scripts/WebCam.cs.meta

@@ -1,11 +0,0 @@
-fileFormatVersion: 2
-guid: 5db51e2552e03de4b9e7e91b5746adbc
-MonoImporter:
-  externalObjects: {}
-  serializedVersion: 2
-  defaultReferences: []
-  executionOrder: 0
-  icon: {instanceID: 0}
-  userData: 
-  assetBundleName: 
-  assetBundleVariant: 

Assets/MediaPipeUnity/Interfaces.meta

@@ -0,0 +1,8 @@
+fileFormatVersion: 2
+guid: 73c615986873dc246893879daf74c05d
+folderAsset: yes
+DefaultImporter:
+  externalObjects: {}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Assets/MediaPipeUnity/Interfaces/Listener.cs

@@ -0,0 +1,14 @@
+using System.Collections;
+using System.Collections.Generic;
+using UnityEngine;
+/// <summary>
+/// Listener interface with an IEnumerator as its processing-function.
+/// </summary>
+public interface Listener
+{
+    /// <summary>
+    /// The function that is called by the publisher.
+    /// </summary>
+    /// <returns></returns>
+    public IEnumerator ProcessIncomingCall();
+}

Assets/MediaPipeUnity/Interfaces/Listener.cs.meta

@@ -1,5 +1,5 @@
 fileFormatVersion: 2
-guid: 44e682a32ee15cc489bf50f3a06f717b
+guid: e4c1da9896d9ba2449549a016b5fd15e
 MonoImporter:
   externalObjects: {}
   serializedVersion: 2

Assets/MediaPipeUnity/Interfaces/SignPredictorInterfaces.asmdef

@@ -0,0 +1,14 @@
+{
+    "name": "SignPredictorInterfaces",
+    "rootNamespace": "",
+    "references": [],
+    "includePlatforms": [],
+    "excludePlatforms": [],
+    "allowUnsafeCode": false,
+    "overrideReferences": false,
+    "precompiledReferences": [],
+    "autoReferenced": true,
+    "defineConstraints": [],
+    "versionDefines": [],
+    "noEngineReferences": false
+}

Assets/MediaPipeUnity/Interfaces/SignPredictorInterfaces.asmdef.meta

@@ -0,0 +1,7 @@
+fileFormatVersion: 2
+guid: f55a02e98b01bc849b30d9650ccd8f15
+AssemblyDefinitionImporter:
+  externalObjects: {}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Assets/MediaPipeUnity/Prefabs/Feedback.prefab

@@ -9,7 +9,6 @@ GameObject:
   serializedVersion: 6
   m_Component:
   - component: {fileID: 4318122119930585316}
-  - component: {fileID: 4318122119930585317}
   m_Layer: 5
   m_Name: Feedback
   m_TagString: Untagged
@@ -39,22 +38,6 @@ RectTransform:
   m_AnchoredPosition: {x: 0, y: 200}
   m_SizeDelta: {x: 500, y: 150}
   m_Pivot: {x: 0.5, y: 0}
---- !u!114 &4318122119930585317
-MonoBehaviour:
-  m_ObjectHideFlags: 0
-  m_CorrespondingSourceObject: {fileID: 0}
-  m_PrefabInstance: {fileID: 0}
-  m_PrefabAsset: {fileID: 0}
-  m_GameObject: {fileID: 4318122119930585319}
-  m_Enabled: 1
-  m_EditorHideFlags: 0
-  m_Script: {fileID: 11500000, guid: 44e682a32ee15cc489bf50f3a06f717b, type: 3}
-  m_Name: 
-  m_EditorClassIdentifier: 
-  feedbackText: {fileID: 4318122120222767928}
-  feedbackProgress: {fileID: 4318122119968934242}
-  feedbackProgressImage: {fileID: 4318122120334233319}
-  signPredictor: {fileID: 0}
 --- !u!1 &4318122119968934244
 GameObject:
   m_ObjectHideFlags: 0

Assets/MediaPipeUnity/ScriptableObjects/ModelList.asset

@@ -17,4 +17,4 @@ MonoBehaviour:
   - index: 0
     model: {fileID: 5022602860645237092, guid: e6d85df707405ad4f97c23b07227ee99, type: 3}
   - index: 1
-    model: {fileID: 5022602860645237092, guid: e6d85df707405ad4f97c23b07227ee99, type: 3}
+    model: {fileID: 0}

Assets/MediaPipeUnity/Scripts/AbstractFeedback.cs

@@ -0,0 +1,40 @@
+using DigitalRuby.Tween;
+using Mediapipe.Unity.Tutorial;
+using System;
+using System.Collections;
+using TMPro;
+using UnityEngine;
+using UnityEngine.Events;
+using UnityEngine.UI;
+
+/// <summary>
+/// Class to display feedback during a course
+/// </summary>
+public abstract class AbstractFeedback : MonoBehaviour, Listener
+{
+
+    /// <summary>
+    /// Reference to the sign predictor
+    /// </summary>
+    public SignPredictor signPredictor;
+    /// <summary>
+    /// The function that is called by the publisher on all its listeners
+    /// </summary>
+    /// <returns></returns>
+    public IEnumerator ProcessIncomingCall()
+    {
+        yield return StartCoroutine(UpdateFeedback());
+    }
+    /// <summary>
+    /// A function to add yourself as listener to the signPredictor you are holding
+    /// </summary>
+    public void AddSelfAsListener()
+    {
+        signPredictor.listeners.Add(this);
+    }
+    /// <summary>
+    /// The function that holds the logic to process the new probabilities of the signPredictor
+    /// </summary>
+    /// <returns></returns>
+    protected abstract IEnumerator UpdateFeedback();
+}

Assets/MediaPipeUnity/Scripts/AbstractFeedback.cs.meta

@@ -1,5 +1,5 @@
 fileFormatVersion: 2
-guid: a62d2b8bded916443835dc19010b83c1
+guid: 7b5ac794337a54143a6e3077483d96c9
 MonoImporter:
   externalObjects: {}
   serializedVersion: 2

Assets/MediaPipeUnity/Scripts/Feedback.cs

@@ -1,182 +0,0 @@
-using DigitalRuby.Tween;
-using Mediapipe.Unity.Tutorial;
-using System;
-using System.Collections;
-using TMPro;
-using UnityEngine;
-using UnityEngine.Events;
-using UnityEngine.UI;
-
-/// <summary>
-/// Class to display feedback during a course
-/// </summary>
-public class Feedback : MonoBehaviour
-{
-    /// <summary>
-    /// Reference to the feedback field
-    /// </summary>
-    public TMP_Text feedbackText;
-
-    /// <summary>
-    /// Reference to the progress bar
-    /// </summary>
-    public Slider feedbackProgress;
-
-    /// <summary>
-    /// Reference to the progress bar image, so we can add fancy colors
-    /// </summary>
-    public Image feedbackProgressImage;
-
-    /// <summary>
-    /// Reference to the sign predictor
-    /// </summary>
-    public SignPredictor signPredictor;
-
-    /// <summary>
-    /// Callback for getting the correct sign
-    /// </summary>
-    public Func<string> getSignCallback;
-
-    /// <summary>
-    /// Callback to initiate the next sign
-    /// </summary>
-    public UnityAction<string> predictSignCallback;
-
-    /// <summary>
-    /// Timer to keep track of how long a incorrect sign is performed
-    /// </summary>
-    private DateTime timer;
-
-    /// <summary>
-    /// Current predicted sign
-    /// </summary>
-    private string predictedSign = null;
-
-    /// <summary>
-    /// Previous incorrect sign, so we can keep track whether the user is wrong or the user is still changing signs
-    /// </summary>
-    private string previousIncorrectSign = null;
-
-    /// <summary>
-    /// Start is called before the first frame update
-    /// </summary>
-    void Start()
-    {
-        // Start the coroutine to update the scale every 200 milliseconds
-        StartCoroutine(UpdateFeedback());
-    }
-
-    /// <summary>
-    /// UpdateScale updates the progress bar every 200ms, updated the feedback text, and progress bar color
-    /// If a high enough accuracy is detected, it will go to the next sign
-    /// </summary>
-    /// <returns></returns>
-    IEnumerator UpdateFeedback()
-    {
-        while (true)
-        {
-            if (getSignCallback != null && predictSignCallback != null)
-            {
-
-                // Get current sign
-                string currentSign = getSignCallback();
-                // Get the predicted sign
-                if (signPredictor != null && signPredictor.learnableProbabilities != null &&
-                    currentSign != null && signPredictor.learnableProbabilities.ContainsKey(currentSign))
-                {    
-                    float accuracy = signPredictor.learnableProbabilities[currentSign];
-                    if (feedbackText != null && feedbackProgressImage != null){
-                        if (accuracy > 0.98)
-                        {
-                            // TODO: fix emojis
-                            feedbackText.text = "✨ Perfect ✨";
-                            Color col = new Color(0xff / 255.0f, 0xcc / 255.0f, 0x00 / 255.0f);
-                            feedbackText.color = col;
-                            feedbackProgressImage.color = col;
-                        }
-                        else if (accuracy > 0.95)
-                        {
-                            feedbackText.text = "Super!";
-                            Color col = new Color(0x00 / 255.0f, 0xff / 255.0f, 0xcc / 255.0f);
-                            feedbackText.color = col;
-                            feedbackProgressImage.color = col;
-                        }
-                        else if (accuracy > 0.90)
-                        {
-                            feedbackText.text = "Goed";
-                            feedbackText.color = Color.green;
-                            feedbackProgressImage.color = Color.green;
-                        }
-                        else if (accuracy > 0.80)
-                        {
-                            feedbackText.text = "Bijna...";
-                            Color col = new Color(0xff / 255.0f, 0x66 / 255.0f, 0x00 / 255.0f);
-                            feedbackText.color = col;
-                            feedbackProgressImage.color = col;
-                        }
-                        else
-                        {
-                            feedbackText.text = "Detecteren...";
-                            feedbackText.color = Color.red;
-                            feedbackProgressImage.color = Color.red;
-                        }
-
-                        float oldValue = feedbackProgress.value;
-                        // use an exponential scale 
-                        float newValue = Mathf.Exp(4 * (accuracy - 1.0f));
-                        feedbackProgress.gameObject.Tween("FeedbackUpdate", oldValue, newValue, 0.2f, TweenScaleFunctions.CubicEaseInOut, (t) =>
-                        {
-                            if (feedbackProgress != null)
-                            {
-                                feedbackProgress.value = t.CurrentValue;
-                            }
-                        });
-                    }
-
-                    // Check whether (in)correct sign has high accuracy
-                    foreach (var kv in signPredictor.learnableProbabilities)
-                    {
-                        if (kv.Value > 0.90)
-                        {
-                            predictedSign = kv.Key;
-                            // Correct sign
-                            if (predictedSign == currentSign)
-                            {
-                                yield return new WaitForSeconds(1.0f);
-                                predictSignCallback(predictedSign);
-                                timer = DateTime.Now;
-                                predictedSign = null;
-                                previousIncorrectSign = null;
-                            }
-                            // Incorrect sign
-                            else
-                            {
-                                if (previousIncorrectSign != predictedSign)
-                                {
-                                    timer = DateTime.Now;
-                                    previousIncorrectSign = predictedSign;
-                                }
-                                else if (DateTime.Now - timer > TimeSpan.FromSeconds(2.0f))
-                                {
-                                    predictSignCallback(predictedSign);
-                                    timer = DateTime.Now;
-                                    predictedSign = null;
-                                    previousIncorrectSign = null;
-                                }
-                            }
-                            break;
-                        }
-                    }
-                }
-                else if(feedbackProgress != null)
-                {
-                    
-                    feedbackProgress.value = 0.0f;
-                }
-            }
-
-            // Wait for 200 milliseconds before updating the scale again
-            yield return new WaitForSeconds(0.2f);
-        }
-    }
-}

Assets/MediaPipeUnity/Scripts/SignPredictor.asmdef

@@ -7,7 +7,8 @@
         "GUID:04c4d86a70aa56c55a78c61f1ab1a56d",
         "GUID:edc93f477bb73a743a97d6882ed330b3",
         "GUID:58e104b97fb3752438ada2902a36dcbf",
-        "GUID:7f2d0ee6dd21e1d4eb25b71b7a749d25"
+        "GUID:7f2d0ee6dd21e1d4eb25b71b7a749d25",
+        "GUID:f55a02e98b01bc849b30d9650ccd8f15"
     ],
     "includePlatforms": [],
     "excludePlatforms": [],

Assets/MediaPipeUnity/Scripts/SignPredictor.cs

@@ -141,6 +141,8 @@ namespace Mediapipe.Unity.Tutorial
         /// </summary>
         private Tensor inputTensor;
 
+        public List<Listener> listeners = new List<Listener>();
+
         /// <summary>
         /// Google Mediapipe setup & run
         /// </summary>
@@ -159,6 +161,7 @@ namespace Mediapipe.Unity.Tutorial
 
             webcamTexture.Play();
 
+            
             yield return new WaitUntil(() => webcamTexture.width > 16);
 
             // Set webcam aspect ratio
@@ -167,63 +170,63 @@ namespace Mediapipe.Unity.Tutorial
             float webcamAspect = (float)webcamTexture.width / (float)webcamTexture.height;
             screen.rectTransform.sizeDelta = new Vector2(screen.rectTransform.sizeDelta.y * webcamAspect, (screen.rectTransform.sizeDelta.y));
             screen.texture = webcamTexture;
-            if(screen2 != null)
+            if (screen2 != null)
             {
                 screen2.rectTransform.sizeDelta = new Vector2(screen2.rectTransform.sizeDelta.y * webcamAspect, (screen2.rectTransform.sizeDelta.y));
             }
 
-            // TODO this method is kinda meh you should use    
+            if (modelList.GetCurrentModel() != null)
-            inputTexture = new Texture2D(width, height, TextureFormat.RGBA32, false);
-            pixelData = new Color32[width * height];
-
-            if (!resourceManagerIsInitialized)
             {
-                resourceManager = new StreamingAssetsResourceManager();
+                // TODO this method is kinda meh you should use    
-                yield return resourceManager.PrepareAssetAsync("pose_detection.bytes");
+                inputTexture = new Texture2D(width, height, TextureFormat.RGBA32, false);
-                yield return resourceManager.PrepareAssetAsync("pose_landmark_full.bytes");
+                pixelData = new Color32[width * height];
-                yield return resourceManager.PrepareAssetAsync("face_landmark.bytes");
+
-                yield return resourceManager.PrepareAssetAsync("hand_landmark_full.bytes");
+                if (!resourceManagerIsInitialized)
-                yield return resourceManager.PrepareAssetAsync("face_detection_short_range.bytes");
+                {
-                yield return resourceManager.PrepareAssetAsync("hand_recrop.bytes");
+                    resourceManager = new StreamingAssetsResourceManager();
-                yield return resourceManager.PrepareAssetAsync("handedness.txt");
+                    yield return resourceManager.PrepareAssetAsync("pose_detection.bytes");
-                resourceManagerIsInitialized = true;
+                    yield return resourceManager.PrepareAssetAsync("pose_landmark_full.bytes");
+                    yield return resourceManager.PrepareAssetAsync("face_landmark.bytes");
+                    yield return resourceManager.PrepareAssetAsync("hand_landmark_full.bytes");
+                    yield return resourceManager.PrepareAssetAsync("face_detection_short_range.bytes");
+                    yield return resourceManager.PrepareAssetAsync("hand_recrop.bytes");
+                    yield return resourceManager.PrepareAssetAsync("handedness.txt");
+                    resourceManagerIsInitialized = true;
+                }
+
+                stopwatch = new Stopwatch();
+
+                // Setting up the graph
+                graph = new CalculatorGraph(configAsset.text);
+
+                posestream = new OutputStream<NormalizedLandmarkListPacket, NormalizedLandmarkList>(graph, "pose_landmarks", "pose_landmarks_presence");
+                leftstream = new OutputStream<NormalizedLandmarkListPacket, NormalizedLandmarkList>(graph, "left_hand_landmarks", "left_hand_landmarks_presence");
+                rightstream = new OutputStream<NormalizedLandmarkListPacket, NormalizedLandmarkList>(graph, "right_hand_landmarks", "right_hand_landmarks_presence");
+
+                posestream.StartPolling().AssertOk();
+                leftstream.StartPolling().AssertOk();
+                rightstream.StartPolling().AssertOk();
+
+                graph.StartRun().AssertOk();
+                stopwatch.Start();
+
+
+                keypointManager = new KeypointManager(modelInfoFile);
+                // check if model exists at path
+                //var model = ModelLoader.Load(Resources.Load<NNModel>("Models/Fingerspelling/model_A-L"));
+                worker = modelList.GetCurrentModel().CreateWorker();
+
+                StartCoroutine(SignRecognitionCoroutine());
+                StartCoroutine(MediapipeCoroutine());
             }
-
-            stopwatch = new Stopwatch();
-
-            // Setting up the graph
-            graph = new CalculatorGraph(configAsset.text);
-
-            posestream = new OutputStream<NormalizedLandmarkListPacket, NormalizedLandmarkList>(graph, "pose_landmarks", "pose_landmarks_presence");
-            leftstream = new OutputStream<NormalizedLandmarkListPacket, NormalizedLandmarkList>(graph, "left_hand_landmarks", "left_hand_landmarks_presence");
-            rightstream = new OutputStream<NormalizedLandmarkListPacket, NormalizedLandmarkList>(graph, "right_hand_landmarks", "right_hand_landmarks_presence");
-
-            posestream.StartPolling().AssertOk();
-            leftstream.StartPolling().AssertOk();
-            rightstream.StartPolling().AssertOk();
-
-            graph.StartRun().AssertOk();
-            stopwatch.Start();
-
-
-            keypointManager = new KeypointManager(modelInfoFile);
-
-            // check if model exists at path
-            //var model = ModelLoader.Load(Resources.Load<NNModel>("Models/Fingerspelling/model_A-L"));
-            worker = modelList.GetCurrentModel().CreateWorker();
-
-            StartCoroutine(SignRecognitionCoroutine());
-            StartCoroutine(MediapipeCoroutine());
         }
-
+        /// <summary>
-        public void ChangeModel(ModelIndex index)
+        /// Called at the start of course/Minigame, will set the model before the start of SIgnPredictor is called.
+        /// </summary>
+        /// <param name="index">The index of the model to be used</param>
+        public void SetModel(ModelIndex index)
         {
             this.modelList.SetCurrentModel(index);
-            // If a worker already existed, we throw it out
-            worker?.Dispose();
-
-            // Add a new worker for the new model
-            worker = modelList.GetCurrentModel().CreateWorker();
         }
 
         /// <summary>
@@ -325,6 +328,10 @@ namespace Mediapipe.Unity.Tutorial
                         learnableProbabilities.Add(((char)(i + 65)).ToString(), softmaxedOutput2[i]);
                     }
                     //Debug.Log($"prob = [{learnableProbabilities.Aggregate("   ", (t, kv) => $"{t}{kv.Key}:{kv.Value}   ")}]");
+                    foreach(Listener listener in listeners)
+                    {
+                        yield return listener.ProcessIncomingCall();
+                    }
                 }
                 else
                 {

Assets/SpellingBee/PlayModeTests/SpellingBeePlayModeTests.asmdef

@@ -7,7 +7,8 @@
         "InterfacesScripts",
         "Unity.TextMeshPro",
         "SpellingBeeScripts",
-        "AccountsScripts"
+        "AccountsScripts",
+        "SignPredictor"
     ],
     "includePlatforms": [],
     "excludePlatforms": [],

Assets/SpellingBee/Scenes/SpellingBeeGame.unity

@@ -1741,13 +1741,35 @@ RectTransform:
   m_PrefabAsset: {fileID: 0}
 --- !u!114 &967164045 stripped
 MonoBehaviour:
-  m_CorrespondingSourceObject: {fileID: 4318122119930585317, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3}
+  m_CorrespondingSourceObject: {fileID: 4318122120334233319, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3}
   m_PrefabInstance: {fileID: 967164043}
   m_PrefabAsset: {fileID: 0}
   m_GameObject: {fileID: 0}
   m_Enabled: 1
   m_EditorHideFlags: 0
-  m_Script: {fileID: 11500000, guid: 44e682a32ee15cc489bf50f3a06f717b, type: 3}
+  m_Script: {fileID: 11500000, guid: fe87c0e1cc204ed48ad3b37840f39efc, type: 3}
+  m_Name: 
+  m_EditorClassIdentifier: 
+--- !u!114 &967164046 stripped
+MonoBehaviour:
+  m_CorrespondingSourceObject: {fileID: 4318122119968934242, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3}
+  m_PrefabInstance: {fileID: 967164043}
+  m_PrefabAsset: {fileID: 0}
+  m_GameObject: {fileID: 0}
+  m_Enabled: 1
+  m_EditorHideFlags: 0
+  m_Script: {fileID: 11500000, guid: 67db9e8f0e2ae9c40bc1e2b64352a6b4, type: 3}
+  m_Name: 
+  m_EditorClassIdentifier: 
+--- !u!114 &967164047 stripped
+MonoBehaviour:
+  m_CorrespondingSourceObject: {fileID: 4318122120222767928, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3}
+  m_PrefabInstance: {fileID: 967164043}
+  m_PrefabAsset: {fileID: 0}
+  m_GameObject: {fileID: 0}
+  m_Enabled: 1
+  m_EditorHideFlags: 0
+  m_Script: {fileID: 11500000, guid: f4688fdb7df04437aeb418b961361dc5, type: 3}
   m_Name: 
   m_EditorClassIdentifier: 
 --- !u!1 &978093274
@@ -3584,6 +3606,7 @@ MonoBehaviour:
   m_Script: {fileID: 11500000, guid: 44fbed5ae228de39b9f727def7578d06, type: 3}
   m_Name: 
   m_EditorClassIdentifier: 
+  signPredictor: {fileID: 1592592444}
   themeList: {fileID: 11400000, guid: a247e2ce790f0f746a3bc521e6ab7d58, type: 2}
   userList: {fileID: 11400000, guid: 072bec636a40f7e4e93b0ac624a3bda2, type: 2}
   minigame: {fileID: 11400000, guid: 8a087d241d652634eb4f6352267ea7dc, type: 2}
@@ -3594,8 +3617,10 @@ MonoBehaviour:
   timerText: {fileID: 1843239267}
   bonusTimeText: {fileID: 1812475780}
   Scoreboard: {fileID: 862382568}
-  feedback: {fileID: 967164045}
   gameEndedPanel: {fileID: 757133117}
+  feedbackText: {fileID: 967164047}
+  feedbackProgress: {fileID: 967164046}
+  feedbackProgressImage: {fileID: 967164045}
 --- !u!1 &1499197558
 GameObject:
   m_ObjectHideFlags: 0
@@ -3957,7 +3982,6 @@ MonoBehaviour:
   m_Name: 
   m_EditorClassIdentifier: 
   modelList: {fileID: 11400000, guid: 39516e4e6e56f0f4f80647d9c4d8034c, type: 2}
-  model: {fileID: 5022602860645237092, guid: e6d85df707405ad4f97c23b07227ee99, type: 3}
   modelInfoFile: {fileID: 4900000, guid: fb8b51022bdcd654a9f29c054832a1b5, type: 3}
   configAsset: {fileID: 4900000, guid: 6288c43cdca97374782dac1ea87aa029, type: 3}
   screen: {fileID: 1743003084}

Assets/SpellingBee/Scripts/SpellingBeeController.cs

@@ -5,8 +5,9 @@ using System.Linq;
 using TMPro;
 using UnityEngine;
 using UnityEngine.UI;
+using DigitalRuby.Tween;
 
-public partial class SpellingBeeController : MonoBehaviour
+public partial class SpellingBeeController : AbstractFeedback
 {
     /// <summary>
     /// All of the words that can be used in this session
@@ -136,20 +137,55 @@ public partial class SpellingBeeController : MonoBehaviour
     /// </summary>
     public Transform Scoreboard;
 
-    /// <summary>
-    /// Accuracy feeback object
-    /// </summary>
-    public Feedback feedback;
-
     /// <summary>
     /// Reference to the gameEnded panel, so we can update its display
     /// </summary>
     public GameObject gameEndedPanel;
 
+    /// <summary>
+    /// Reference to the feedback field
+    /// </summary>
+    public TMP_Text feedbackText;
+
+    /// <summary>
+    /// Reference to the progress bar
+    /// </summary>
+    public Slider feedbackProgress;
+
+    /// <summary>
+    /// Reference to the progress bar image, so we can add fancy colors
+    /// </summary>
+    public Image feedbackProgressImage;
+
+    /// <summary>
+    /// Timer to keep track of how long a incorrect sign is performed
+    /// </summary>
+    protected DateTime timer;
+
+    /// <summary>
+    /// Current predicted sign
+    /// </summary>
+    protected string predictedSign = null;
+
+    /// <summary>
+    /// Previous incorrect sign, so we can keep track whether the user is wrong or the user is still changing signs
+    /// </summary>
+    protected string previousIncorrectSign = null;
+
     /// <summary>
     /// Start is called before the first frame update
     /// </summary>
     public void Start()
+    {
+        StartController();
+
+        signPredictor.SetModel(currentTheme.modelIndex);
+        AddSelfAsListener();
+    }
+    /// <summary>
+    /// Is called at the start of the scene AND when the game is replayed
+    /// </summary>
+    public void StartController()
     {
         correctLetters = 0;
         incorrectLetters = 0;
@@ -182,29 +218,10 @@ public partial class SpellingBeeController : MonoBehaviour
         userList.Save();
 
         currentTheme = minigame.themeList.themes[minigame.themeList.currentThemeIndex];
-        feedback.signPredictor.ChangeModel(currentTheme.modelIndex);
+        //feedback.signPredictor.ChangeModel(currentTheme.modelIndex);
         words.AddRange(currentTheme.learnables);
         ShuffleWords();
         NextWord();
-
-        // Set calllbacks
-        feedback.getSignCallback = () =>
-        {
-            if (letterIndex < currentWord.Length)
-            {
-                return currentWord[letterIndex].ToString().ToUpper();
-            }
-            return null;
-        };
-        feedback.predictSignCallback = (sign) =>
-        {
-            bool successful = sign.ToUpper() == currentWord[letterIndex].ToString().ToUpper();
-            if (successful)
-            {
-                AddSeconds(secondsPerLetter);
-            }
-            NextLetter(successful);
-        };
     }
 
     /// <summary>
@@ -453,4 +470,116 @@ public partial class SpellingBeeController : MonoBehaviour
     {
         yield return new WaitForSecondsRealtime(2);
     }
+
+    /// <summary>
+    /// The updateFunction that is called when new probabilities become available
+    /// </summary>
+    /// <returns></returns>
+    protected override IEnumerator UpdateFeedback()
+    {
+        // Get current sign
+        string currentSign = GetSign();
+        // Get the predicted sign
+        if (signPredictor != null && signPredictor.learnableProbabilities != null &&
+            currentSign != null && signPredictor.learnableProbabilities.ContainsKey(currentSign))
+        {
+            float accuracy = signPredictor.learnableProbabilities[currentSign];
+            if (feedbackText != null && feedbackProgressImage != null)
+            {
+                if (accuracy > 0.90)
+                {
+                    feedbackText.text = "Goed";
+                    feedbackText.color = Color.green;
+                    feedbackProgressImage.color = Color.green;
+                }
+                else if (accuracy > 0.80)
+                {
+                    feedbackText.text = "Bijna...";
+                    Color col = new Color(0xff / 255.0f, 0x66 / 255.0f, 0x00 / 255.0f);
+                    feedbackText.color = col;
+                    feedbackProgressImage.color = col;
+                }
+                else
+                {
+                    feedbackText.text = "Detecteren...";
+                    feedbackText.color = Color.red;
+                    feedbackProgressImage.color = Color.red;
+                }
+
+                float oldValue = feedbackProgress.value;
+                // use an exponential scale 
+                float newValue = Mathf.Exp(4 * (accuracy - 1.0f));
+                feedbackProgress.gameObject.Tween("FeedbackUpdate", oldValue, newValue, 0.2f, TweenScaleFunctions.CubicEaseInOut, (t) =>
+                {
+                    if (feedbackProgress != null)
+                    {
+                        feedbackProgress.value = t.CurrentValue;
+                    }
+                });
+            }
+
+            // Check whether (in)correct sign has high accuracy
+            foreach (var kv in signPredictor.learnableProbabilities)
+            {
+                if (kv.Value > 0.90)
+                {
+                    predictedSign = kv.Key;
+                    // Correct sign
+                    if (predictedSign == currentSign)
+                    {
+                        yield return new WaitForSeconds(1.0f);
+                        predictSign(predictedSign);
+                        timer = DateTime.Now;
+                        predictedSign = null;
+                        previousIncorrectSign = null;
+                    }
+                    // Incorrect sign
+                    else
+                    {
+                        if (previousIncorrectSign != predictedSign)
+                        {
+                            timer = DateTime.Now;
+                            previousIncorrectSign = predictedSign;
+                        }
+                        else if (DateTime.Now - timer > TimeSpan.FromSeconds(2.0f))
+                        {
+                            predictSign(predictedSign);
+                            timer = DateTime.Now;
+                            predictedSign = null;
+                            previousIncorrectSign = null;
+                        }
+                    }
+                    break;
+                }
+            }
+        }
+        else if (feedbackProgress != null)
+        {
+
+            feedbackProgress.value = 0.0f;
+        }
+        yield return null;
+    }
+    /// <summary>
+    /// Function to get the current letter that needs to be signed
+    /// </summary>
+    /// <returns>the current letter that needs to be signed</returns>
+    public string GetSign(){
+        if (letterIndex<currentWord.Length){
+            return currentWord[letterIndex].ToString().ToUpper();
+        }
+        return null;
+    }
+    /// <summary>
+    /// Function to confirm your prediction and check if it is correct. 
+    /// </summary>
+    /// <param name="sign"></param>
+    public void predictSign(string sign) { 
+        bool successful = sign.ToUpper() == currentWord[letterIndex].ToString().ToUpper();
+        if (successful)
+        {
+            AddSeconds(secondsPerLetter);
+        }
+        NextLetter(successful);
+    }
 }

Assets/SpellingBee/Scripts/SpellingBeeController.cs.meta

@@ -3,21 +3,7 @@ guid: 44fbed5ae228de39b9f727def7578d06
 MonoImporter:
   externalObjects: {}
   serializedVersion: 2
-  defaultReferences:
+  defaultReferences: []
-  - input: {instanceID: 0}
-  - endText: {instanceID: 0}
-  - correctWordsText: {instanceID: 0}
-  - correctLettersText: {instanceID: 0}
-  - gameEndedPanel: {instanceID: 0}
-  - replayButton: {instanceID: 0}
-  - userList: {instanceID: 0}
-  - minigame: {instanceID: 0}
-  - letterPrefab: {instanceID: 0}
-  - letterContainer: {instanceID: 0}
-  - wordImage: {instanceID: 0}
-  - timerText: {instanceID: 0}
-  - Scoreboard: {instanceID: 0}
-  - scoreboardEntry: {fileID: 9154151134820372555, guid: d4a3a228b08d61847acc6da35b44e52c, type: 3}
   executionOrder: 0
   icon: {instanceID: 0}
   userData: 

Assets/SpellingBee/Scripts/SpellingBeeScripts.asmdef

@@ -6,8 +6,8 @@
         "GUID:1631ed2680c61245b8211d943c1639a8",
         "GUID:3444c67d5a3a93e5a95a48906078c372",
         "GUID:d0b6b39a21908f94fbbd9f2c196a9725",
-        "GUID:5c2b5ba89f9e74e418232e154bc5cc7a",
+        "GUID:7f2d0ee6dd21e1d4eb25b71b7a749d25",
-        "GUID:7f2d0ee6dd21e1d4eb25b71b7a749d25"
+        "GUID:58e104b97fb3752438ada2902a36dcbf"
     ],
     "includePlatforms": [],
     "excludePlatforms": [],