deleted stale dependency

Removed items in abstract class that don't need to be there

Assets/Common/Interfaces/ModelIndex.cs

@@ -0,0 +1,11 @@
+using System.Collections;
+using System.Collections.Generic;
+using UnityEngine;
+/// <summary>
+/// This enum is used to identify each of the SignLanguage models
+/// </summary>
+public enum ModelIndex 
+{
+    FINGERSPELLING,
+    NONE
+}

Assets/Common/Interfaces/ModelIndex.cs.meta

@@ -1,5 +1,5 @@
 fileFormatVersion: 2
-guid: 44e682a32ee15cc489bf50f3a06f717b
+guid: 6dbd5e1100bc81648b52206df369d0a1
 MonoImporter:
   externalObjects: {}
   serializedVersion: 2

Assets/Common/Interfaces/ModelList.cs

@@ -0,0 +1,57 @@
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using UnityEngine;
+using Unity.Barracuda;
+/// <summary>
+/// This scriptable will hold tupples of Courseindices and models
+/// </summary>
+[CreateAssetMenu(menuName = "Create new Scriptable/ModelList")]
+public class ModelList : ScriptableObject
+{
+    /// <summary>
+    /// Small class to link a model to a courseIndex irrespective of its position in a list
+    /// </summary>
+    [Serializable]
+    public class ModelTuple
+    {
+        /// <summary>
+        /// ModelIndex to which the model corresponds
+        /// </summary>
+        public ModelIndex index;
+        /// <summary>
+        /// The model itself
+        /// </summary>
+        public NNModel model;
+    }
+
+
+    /// <summary>
+    /// Index of the currently active model
+    /// </summary>
+    public int currentModelIndex = 0;
+
+    /// <summary>
+    /// A list of all the models
+    /// </summary>
+    public List<ModelTuple> models = new List<ModelTuple>();
+
+    /// <summary>
+    /// Get a model by modelindex
+    /// </summary>
+    /// <param name="modelIndex">ModelIndex of the model</param>
+    /// <returns>Model associated with this index, null if no model was found</returns>
+    public NNModel GetCurrentModel()
+    {
+        return models.Find(x => x.model == models[currentModelIndex].model)?.model;
+    }
+
+    /// <summary>
+    /// Function to find a model-index in the list based on its index
+    /// </summary>
+    /// <param name="title"></param>
+    public void SetCurrentModel(ModelIndex index)
+    {
+        currentModelIndex = models.FindIndex((m) => m.index == index);
+    }
+}

Assets/Common/Interfaces/ModelList.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: 78a3f61c93a08c04496c49ffd10b9021
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Assets/Common/Interfaces/Theme.cs

@@ -1,5 +1,4 @@
 using System.Collections.Generic;
-using Unity.Barracuda;
 using UnityEngine;
 
 /// <summary>
@@ -24,9 +23,9 @@ public class Theme : ScriptableObject
     public ThemeIndex index;
 
     /// <summary>
-    /// Reference to the model used in the SignPredictor
+    /// The index of the model you want to use
     /// </summary>
-    public NNModel model;
+    public ModelIndex modelIndex;
 
     /// <summary>
     /// List of all learnable words/letters

Assets/Common/PlayModeTests/CommonPlayModeTests.asmdef

@@ -8,7 +8,8 @@
         "CommonScripts",
         "InterfacesScripts",
         "Unity.TextMeshPro",
-        "AccountsScripts"
+        "AccountsScripts",
+        "SignPredictor"
     ],
     "includePlatforms": [],
     "excludePlatforms": [],

Assets/Common/ScriptableObjects/AnimalsTheme.asset

@@ -16,6 +16,7 @@ MonoBehaviour:
   description: Van vis tot leeuw
   index: 2
   model: {fileID: 0}
+  modelIndex: 1
   learnables:
   - name: Walvis
     image: {fileID: 21300000, guid: 2b01165a5836ab14593d7a5862bd6793, type: 3}

Assets/Common/ScriptableObjects/ClothesTheme.asset

@@ -15,6 +15,8 @@ MonoBehaviour:
   title: Kleren en Kleuren
   description: Van rok tot sok
   index: 1
+  model: {fileID: 0}
+  modelIndex: 1
   learnables:
   - name: Blauw
     image: {fileID: 21300000, guid: 182fb89eba9c64041bef31ca35c4bcd8, type: 3}

Assets/Common/ScriptableObjects/FamilyTheme.asset

@@ -15,6 +15,8 @@ MonoBehaviour:
   title: Familie
   description: Van generatie tot generatie
   index: 6
+  model: {fileID: 0}
+  modelIndex: 1
   learnables:
   - name: Broer
     image: {fileID: 21300000, guid: eecf67266f150f1489717049489cf16d, type: 3}

Assets/Common/ScriptableObjects/FoodTheme.asset

@@ -16,6 +16,7 @@ MonoBehaviour:
   description: Van kers tot pompoen
   index: 3
   model: {fileID: 0}
+  modelIndex: 1
   learnables:
   - name: Aardappel
     image: {fileID: 21300000, guid: 2610cdbc24a125f43ada7fed67d8f51b, type: 3}

Assets/Common/ScriptableObjects/HobbyTheme.asset

@@ -15,6 +15,8 @@ MonoBehaviour:
   title: Hobbies
   description: Van schilderen tot reizen
   index: 4
+  model: {fileID: 0}
+  modelIndex: 1
   learnables:
   - name: Dansen
     image: {fileID: 21300000, guid: 6d405f607ae817744b49f921f0611088, type: 3}

Assets/Common/ScriptableObjects/HouseTheme.asset

@@ -15,6 +15,8 @@ MonoBehaviour:
   title: Huis beschrijven
   description: Van zetel tot villa
   index: 5
+  model: {fileID: 0}
+  modelIndex: 1
   learnables:
   - name: Keuken
     image: {fileID: 21300000, guid: b17ce5bf59092b847b084d3400e7a1b4, type: 3}

Assets/Common/Scripts/CommonScripts.asmdef

@@ -5,8 +5,8 @@
         "GUID:6055be8ebefd69e48b49212b09b47b2f",
         "GUID:63c63e721f65ebb7d871cb9ef49f4752",
         "GUID:1631ed2680c61245b8211d943c1639a8",
-        "GUID:5c2b5ba89f9e74e418232e154bc5cc7a",
-        "GUID:7f2d0ee6dd21e1d4eb25b71b7a749d25"
+        "GUID:7f2d0ee6dd21e1d4eb25b71b7a749d25",
+        "GUID:d0b6b39a21908f94fbbd9f2c196a9725"
     ],
     "includePlatforms": [],
     "excludePlatforms": [],

Assets/Common/Scripts/CourseActivityScreen.cs

@@ -56,8 +56,8 @@ public class CourseActivityScreen : MonoBehaviour
         Course course = courseList.courses[index];
 
         // vvv TEMPORARY STUFF vvv
-        playButton.SetActive(course.theme.model != null);
-        previewButton.SetActive(course.theme.model == null);
+        playButton.SetActive(course.theme.modelIndex != ModelIndex.NONE);
+        previewButton.SetActive(course.theme.modelIndex == ModelIndex.NONE);
         // ^^^ TEMPORARY STUFF ^^^
 
         title.text = course.title;

Assets/Common/Tests/CommonTests.asmdef

@@ -5,7 +5,9 @@
         "UnityEngine.TestRunner",
         "UnityEditor.TestRunner",
         "CommonScripts",
-        "InterfacesScripts"
+        "InterfacesScripts",
+        "Unity.Barracuda",
+        "SignPredictor"
     ],
     "includePlatforms": [
         "Editor"

Assets/Common/Tests/ModelListTest.cs

@@ -0,0 +1,80 @@
+using NUnit.Framework;
+using Unity.Barracuda;
+using UnityEngine;
+/// <summary>
+/// Test the ModelList class
+/// </summary>
+[TestFixture]
+public class ModelListTest
+{
+    private ModelList modelList;
+
+    /// <summary>
+    /// Setup a ModelList with all possible Models in the enum
+    /// </summary>
+    [SetUp]
+    public void Setup_Model()
+    {
+        modelList = ScriptableObject.CreateInstance<ModelList>();
+
+        // Add a Model for each index in the enum
+
+        // Dumb way to access each index in the enum, couldn't find a different way to do it though
+        foreach (var field in typeof(ModelIndex).GetFields())
+        {
+            if (field.IsLiteral)
+            {
+                ModelIndex value = (ModelIndex)field.GetValue(null);
+                string name = field.Name;
+                ModelList.ModelTuple model = new ModelList.ModelTuple();
+                // This is all we will need to distinguish 
+                model.index = value;
+
+                // Insert in front to guarantee that ModelIndex will not line up with listIndex
+                modelList.models.Insert(0, model);
+            }
+        }
+    }
+    /// <summary>
+    /// Check if current model can be correctly gotten as current via GetCurrentModel
+    /// </summary>
+    [Test]
+    public void TestGetCurrentModel()
+    {
+        System.Random random = new System.Random();
+        ModelIndex value = (ModelIndex)random.Next(modelList.models.Count);
+        modelList.SetCurrentModel(value);
+
+        Assert.AreEqual(modelList.models[modelList.currentModelIndex].model, modelList.GetCurrentModel());
+
+        // Check if empty model fails gracefully (returns null)
+        Assert.IsNull(ScriptableObject.CreateInstance<ModelList>().GetCurrentModel());
+    }
+
+    /// <summary>
+    /// Check if all models can be correctly set as current via SetCurrentModel
+    /// </summary>
+    [Test]
+    public void TestSetCurrentModel()
+    {
+        foreach (var field in typeof(ModelIndex).GetFields())
+        {
+            if (field.IsLiteral)
+            {
+                ModelIndex value = (ModelIndex)field.GetValue(null);
+                string name = field.Name;
+                modelList.SetCurrentModel(value);
+
+                // Fetch the current model and check if its name is the same as the one we made into the current one 
+                ModelList.ModelTuple m = modelList.models[modelList.currentModelIndex];
+
+                Assert.AreEqual(m.index, value);
+                Assert.IsTrue(m.model is NNModel || m.model is null);
+            }
+        }
+        ModelList emptyList = ScriptableObject.CreateInstance<ModelList>();
+        emptyList.SetCurrentModel(ModelIndex.FINGERSPELLING);
+
+        Assert.IsTrue(emptyList.currentModelIndex == -1);
+    }
+}

Assets/Common/Tests/ModelListTest.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: 1313e0cc80244354eb6e2d0c1e891941
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Assets/Courses/Scenes/TemplateCourse.unity

@@ -490,6 +490,10 @@ MonoBehaviour:
   m_Script: {fileID: 11500000, guid: 6b3f784c065813a4a8364b1299284816, type: 3}
   m_Name: 
   m_EditorClassIdentifier: 
+  feedbackText: {fileID: 4318122121437849762}
+  feedbackProgress: {fileID: 4318122121437849761}
+  feedbackProgressImage: {fileID: 4318122121437849760}
+  signPredictor: {fileID: 883853268}
   previewModel: {fileID: 5022602860645237092, guid: e6d85df707405ad4f97c23b07227ee99, type: 3}
   feedbackProgressBar: {fileID: 4318122121437849759}
   previewMessage: {fileID: 2070775951}
@@ -505,7 +509,6 @@ MonoBehaviour:
   ResultsDecription: {fileID: 100123246}
   CoursesButton: {fileID: 839294691}
   timeSpent: {fileID: 77614869}
-  feedback: {fileID: 1714882683}
 --- !u!1 &361280475
 GameObject:
   m_ObjectHideFlags: 0
@@ -1222,8 +1225,8 @@ MonoBehaviour:
   m_OnClick:
     m_PersistentCalls:
       m_Calls:
-      - m_Target: {fileID: 301088551}
-        m_TargetAssemblyTypeName: TemplateCourse, Assembly-CSharp
+      - m_Target: {fileID: 1335886461}
+        m_TargetAssemblyTypeName: BackButton, CommonScripts
         m_MethodName: Back
         m_Mode: 1
         m_Arguments:
@@ -1301,7 +1304,7 @@ MonoBehaviour:
   m_Script: {fileID: 11500000, guid: 043ccd99cf82b3cc9bf2e00956ce2b93, type: 3}
   m_Name: 
   m_EditorClassIdentifier: 
-  model: {fileID: 5022602860645237092, guid: e6d85df707405ad4f97c23b07227ee99, type: 3}
+  modelList: {fileID: 11400000, guid: 39516e4e6e56f0f4f80647d9c4d8034c, type: 2}
   modelInfoFile: {fileID: 4900000, guid: fb8b51022bdcd654a9f29c054832a1b5, type: 3}
   configAsset: {fileID: 4900000, guid: 6288c43cdca97374782dac1ea87aa029, type: 3}
   screen: {fileID: 378145456}
@@ -1839,6 +1842,17 @@ RectTransform:
   m_CorrespondingSourceObject: {fileID: 8299246693487308515, guid: 3bccdf365a4fbea4d8fa1aa461d3dc5c, type: 3}
   m_PrefabInstance: {fileID: 1335886459}
   m_PrefabAsset: {fileID: 0}
+--- !u!114 &1335886461 stripped
+MonoBehaviour:
+  m_CorrespondingSourceObject: {fileID: 4518652150503380115, guid: 3bccdf365a4fbea4d8fa1aa461d3dc5c, type: 3}
+  m_PrefabInstance: {fileID: 1335886459}
+  m_PrefabAsset: {fileID: 0}
+  m_GameObject: {fileID: 0}
+  m_Enabled: 1
+  m_EditorHideFlags: 0
+  m_Script: {fileID: 11500000, guid: c3dd279b546423e4a8a1b28819a6c4a1, type: 3}
+  m_Name: 
+  m_EditorClassIdentifier: 
 --- !u!1 &1383144366
 GameObject:
   m_ObjectHideFlags: 0
@@ -2252,17 +2266,6 @@ RectTransform:
   m_CorrespondingSourceObject: {fileID: 4318122119930585316, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3}
   m_PrefabInstance: {fileID: 4318122121437849758}
   m_PrefabAsset: {fileID: 0}
---- !u!114 &1714882683 stripped
-MonoBehaviour:
-  m_CorrespondingSourceObject: {fileID: 4318122119930585317, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3}
-  m_PrefabInstance: {fileID: 4318122121437849758}
-  m_PrefabAsset: {fileID: 0}
-  m_GameObject: {fileID: 4318122121437849759}
-  m_Enabled: 1
-  m_EditorHideFlags: 0
-  m_Script: {fileID: 11500000, guid: 44e682a32ee15cc489bf50f3a06f717b, type: 3}
-  m_Name: 
-  m_EditorClassIdentifier: 
 --- !u!1 &1773033262
 GameObject:
   m_ObjectHideFlags: 0
@@ -2838,6 +2841,10 @@ PrefabInstance:
       propertyPath: m_LocalEulerAnglesHint.z
       value: 0
       objectReference: {fileID: 0}
+    - target: {fileID: 4318122119930585317, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3}
+      propertyPath: m_Enabled
+      value: 0
+      objectReference: {fileID: 0}
     - target: {fileID: 4318122119930585317, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3}
       propertyPath: signPredictor
       value: 
@@ -2865,3 +2872,36 @@ GameObject:
   m_CorrespondingSourceObject: {fileID: 4318122119930585319, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3}
   m_PrefabInstance: {fileID: 4318122121437849758}
   m_PrefabAsset: {fileID: 0}
+--- !u!114 &4318122121437849760 stripped
+MonoBehaviour:
+  m_CorrespondingSourceObject: {fileID: 4318122120334233319, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3}
+  m_PrefabInstance: {fileID: 4318122121437849758}
+  m_PrefabAsset: {fileID: 0}
+  m_GameObject: {fileID: 0}
+  m_Enabled: 1
+  m_EditorHideFlags: 0
+  m_Script: {fileID: 11500000, guid: fe87c0e1cc204ed48ad3b37840f39efc, type: 3}
+  m_Name: 
+  m_EditorClassIdentifier: 
+--- !u!114 &4318122121437849761 stripped
+MonoBehaviour:
+  m_CorrespondingSourceObject: {fileID: 4318122119968934242, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3}
+  m_PrefabInstance: {fileID: 4318122121437849758}
+  m_PrefabAsset: {fileID: 0}
+  m_GameObject: {fileID: 0}
+  m_Enabled: 1
+  m_EditorHideFlags: 0
+  m_Script: {fileID: 11500000, guid: 67db9e8f0e2ae9c40bc1e2b64352a6b4, type: 3}
+  m_Name: 
+  m_EditorClassIdentifier: 
+--- !u!114 &4318122121437849762 stripped
+MonoBehaviour:
+  m_CorrespondingSourceObject: {fileID: 4318122120222767928, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3}
+  m_PrefabInstance: {fileID: 4318122121437849758}
+  m_PrefabAsset: {fileID: 0}
+  m_GameObject: {fileID: 0}
+  m_Enabled: 1
+  m_EditorHideFlags: 0
+  m_Script: {fileID: 11500000, guid: f4688fdb7df04437aeb418b961361dc5, type: 3}
+  m_Name: 
+  m_EditorClassIdentifier: 

Assets/Courses/Scripts/CourseScripts.asmdef

@@ -6,7 +6,9 @@
         "AccountsScripts",
         "InterfacesScripts",
         "SignPredictor",
-        "Unity.Barracuda"
+        "Unity.Barracuda",
+        "Tween",
+        "SignPredictorInterfaces"
     ],
     "includePlatforms": [],
     "excludePlatforms": [],

Assets/Courses/Scripts/TemplateCourse.cs

@@ -1,14 +1,16 @@
 using System;
+using System.Collections;
 using TMPro;
 using Unity.Barracuda;
 using UnityEngine;
 using UnityEngine.UI;
 using UnityEngine.Video;
+using DigitalRuby.Tween;
 
 /// <summary>
 /// TemplateCourse scene manager
 /// </summary>
-public class TemplateCourse : MonoBehaviour
+public class TemplateCourse : AbstractFeedback
 {
     // vvv TEMPORARY STUFF vvv
     public NNModel previewModel;
@@ -112,10 +114,37 @@ public class TemplateCourse : MonoBehaviour
     /// </summary>
     public TMP_Text timeSpent;
 
+
     /// <summary>
-    /// Reference to the feedback script on the Feedback prefab
+    /// Reference to the feedback field
     /// </summary>
-    public Feedback feedback;
+    public TMP_Text feedbackText;
+
+    /// <summary>
+    /// Reference to the progress bar
+    /// </summary>
+    public Slider feedbackProgress;
+
+    /// <summary>
+    /// Reference to the progress bar image, so we can add fancy colors
+    /// </summary>
+    public Image feedbackProgressImage;
+
+    /// <summary>
+    /// Timer to keep track of how long a incorrect sign is performed
+    /// </summary>
+    protected DateTime timer;
+
+    /// <summary>
+    /// Current predicted sign
+    /// </summary>
+    protected string predictedSign = null;
+
+    /// <summary>
+    /// Previous incorrect sign, so we can keep track whether the user is wrong or the user is still changing signs
+    /// </summary>
+    protected string previousIncorrectSign = null;
+
 
     /// <summary>
     /// This function is called when the script is initialised.
@@ -124,17 +153,23 @@ public class TemplateCourse : MonoBehaviour
     /// Then it checks whether or not the User has started the course yet, to possibly create a new progress atribute for the course.
     /// Then it sets up the course-screen to display relevant information from the course-scriptable.
     /// </summary>
-    void Awake()
+    void Start()
+    {
+        StartGameController();
+
+        signPredictor.SetModel(course.theme.modelIndex);
+        AddSelfAsListener();
+    }
+
+    public void StartGameController()
     {
         // Setting up course
         course = courselist.courses[courselist.currentCourseIndex];
-        feedback.signPredictor.model = course.theme.model;
         maxWords = course.theme.learnables.Count;
 
         // vvv TEMPORARY STUFF vvv
-        feedbackProgressBar.SetActive(course.theme.model != null);
-        previewMessage.SetActive(course.theme.model == null);
-        feedback.signPredictor.model = previewModel;
+        feedbackProgressBar.SetActive(course.theme.modelIndex != ModelIndex.NONE);
+        previewMessage.SetActive(course.theme.modelIndex == ModelIndex.NONE);
         // ^^^ TEMPORARY STUFF ^^^
 
         // Create entry in current user for keeping track of progress
@@ -163,23 +198,6 @@ public class TemplateCourse : MonoBehaviour
         ResultPanel.SetActive(false);
         // Set the startTime
         startMoment = DateTime.Now;
-
-        // Set callbacks
-        feedback.getSignCallback = () =>
-        {
-            if (currentWordIndex < course.theme.learnables.Count)
-            {
-                return course.theme.learnables[currentWordIndex].name;
-            }
-            return null;
-        };
-        feedback.predictSignCallback = (sign) =>
-        {
-            if (sign == course.theme.learnables[currentWordIndex].name)
-            {
-                NextSign();
-            }
-        };
     }
 
     /// <summary>
@@ -286,4 +304,98 @@ public class TemplateCourse : MonoBehaviour
         progress.AddOrUpdate<float>("courseProgress", 1f);
         userList.Save();
     }
+
+    protected override IEnumerator UpdateFeedback()
+    {
+        // Get current sign
+        string currentSign = course.theme.learnables[currentWordIndex].name;
+        // Get the predicted sign
+        if (signPredictor != null && signPredictor.learnableProbabilities != null &&
+            currentSign != null && signPredictor.learnableProbabilities.ContainsKey(currentSign))
+        {
+            float accuracy = signPredictor.learnableProbabilities[currentSign];
+            if (feedbackText != null && feedbackProgressImage != null)
+            {
+                if (accuracy > 0.90)
+                {
+                    feedbackText.text = "Goed";
+                    feedbackText.color = Color.green;
+                    feedbackProgressImage.color = Color.green;
+                }
+                else if (accuracy > 0.80)
+                {
+                    feedbackText.text = "Bijna...";
+                    Color col = new Color(0xff / 255.0f, 0x66 / 255.0f, 0x00 / 255.0f);
+                    feedbackText.color = col;
+                    feedbackProgressImage.color = col;
+                }
+                else
+                {
+                    feedbackText.text = "Detecteren...";
+                    feedbackText.color = Color.red;
+                    feedbackProgressImage.color = Color.red;
+                }
+
+                float oldValue = feedbackProgress.value;
+                // use an exponential scale 
+                float newValue = Mathf.Exp(4 * (accuracy - 1.0f));
+                feedbackProgress.gameObject.Tween("FeedbackUpdate", oldValue, newValue, 0.2f, TweenScaleFunctions.CubicEaseInOut, (t) =>
+                {
+                    if (feedbackProgress != null)
+                    {
+                        feedbackProgress.value = t.CurrentValue;
+                    }
+                });
+            }
+
+            // Check whether (in)correct sign has high accuracy
+            foreach (var kv in signPredictor.learnableProbabilities)
+            {
+                if (kv.Value > 0.90)
+                {
+                    predictedSign = kv.Key;
+                    // Correct sign
+                    if (predictedSign == currentSign)
+                    {
+                        yield return new WaitForSeconds(1.0f);
+                        CheckEquality(predictedSign);
+                        timer = DateTime.Now;
+                        predictedSign = null;
+                        previousIncorrectSign = null;
+                    }
+                    // Incorrect sign
+                    else
+                    {
+                        if (previousIncorrectSign != predictedSign)
+                        {
+                            timer = DateTime.Now;
+                            previousIncorrectSign = predictedSign;
+                        }
+                        else if (DateTime.Now - timer > TimeSpan.FromSeconds(2.0f))
+                        {
+                            CheckEquality(predictedSign);
+                            timer = DateTime.Now;
+                            predictedSign = null;
+                            previousIncorrectSign = null;
+                        }
+                    }
+                    break;
+                }
+            }
+        }
+        else if (feedbackProgress != null)
+        {
+
+            feedbackProgress.value = 0.0f;
+        }
+        
+    }
+
+    private void CheckEquality(string predicted)
+    {
+        if(predicted == course.theme.learnables[currentWordIndex].name)
+        {
+            NextSign();
+        }
+    }
 }

Assets/Hangman/Scenes/Hangman.unity

@@ -2570,7 +2570,7 @@ Transform:
   m_ConstrainProportionsScale: 0
   m_Children: []
   m_Father: {fileID: 0}
-  m_RootOrder: 4
+  m_RootOrder: 5
   m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
 --- !u!1 &896343212
 GameObject:
@@ -3507,7 +3507,7 @@ RectTransform:
   - {fileID: 1892638588}
   - {fileID: 56162990}
   m_Father: {fileID: 0}
-  m_RootOrder: 2
+  m_RootOrder: 3
   m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
   m_AnchorMin: {x: 0, y: 0}
   m_AnchorMax: {x: 0, y: 0}
@@ -3526,6 +3526,10 @@ MonoBehaviour:
   m_Script: {fileID: 11500000, guid: 2db44635e0eb1e9429a2e6195785364d, type: 3}
   m_Name: 
   m_EditorClassIdentifier: 
+  feedbackText: {fileID: 5233312448025626833}
+  feedbackProgress: {fileID: 5233312447201393291}
+  feedbackProgressImage: {fileID: 5233312447919013134}
+  signPredictor: {fileID: 1991376311}
   themelist: {fileID: 11400000, guid: a247e2ce790f0f746a3bc521e6ab7d58, type: 2}
   letterPrefab: {fileID: 4639383499500021565, guid: c3e66e8957864914cb022af914df6a28, type: 3}
   letterContainer: {fileID: 1870283439}
@@ -3550,7 +3554,6 @@ MonoBehaviour:
   Scoreboard: {fileID: 1007532375}
   EntriesGrid: {fileID: 1391137944}
   scoreboardEntry: {fileID: 9154151134820372555, guid: d4a3a228b08d61847acc6da35b44e52c, type: 3}
-  feedback: {fileID: 5233312447513285388}
   gottogamebutton: {fileID: 1581633295}
 --- !u!1001 &1290865991
 PrefabInstance:
@@ -6245,7 +6248,7 @@ MonoBehaviour:
   m_Script: {fileID: 11500000, guid: 043ccd99cf82b3cc9bf2e00956ce2b93, type: 3}
   m_Name: 
   m_EditorClassIdentifier: 
-  model: {fileID: 5022602860645237092, guid: e6d85df707405ad4f97c23b07227ee99, type: 3}
+  modelList: {fileID: 11400000, guid: 39516e4e6e56f0f4f80647d9c4d8034c, type: 2}
   modelInfoFile: {fileID: 4900000, guid: fb8b51022bdcd654a9f29c054832a1b5, type: 3}
   configAsset: {fileID: 4900000, guid: 6288c43cdca97374782dac1ea87aa029, type: 3}
   screen: {fileID: 1649505745}
@@ -6263,7 +6266,7 @@ Transform:
   m_ConstrainProportionsScale: 0
   m_Children: []
   m_Father: {fileID: 0}
-  m_RootOrder: 5
+  m_RootOrder: 2
   m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
 --- !u!1 &2001212056
 GameObject:
@@ -6566,22 +6569,6 @@ GameObject:
   m_NavMeshLayer: 0
   m_StaticEditorFlags: 0
   m_IsActive: 1
---- !u!114 &5233312447513285388
-MonoBehaviour:
-  m_ObjectHideFlags: 0
-  m_CorrespondingSourceObject: {fileID: 0}
-  m_PrefabInstance: {fileID: 0}
-  m_PrefabAsset: {fileID: 0}
-  m_GameObject: {fileID: 5233312447513285390}
-  m_Enabled: 1
-  m_EditorHideFlags: 0
-  m_Script: {fileID: 11500000, guid: 44e682a32ee15cc489bf50f3a06f717b, type: 3}
-  m_Name: 
-  m_EditorClassIdentifier: 
-  feedbackText: {fileID: 0}
-  feedbackProgress: {fileID: 0}
-  feedbackProgressImage: {fileID: 0}
-  signPredictor: {fileID: 1991376311}
 --- !u!224 &5233312447513285389
 RectTransform:
   m_ObjectHideFlags: 0
@@ -6597,7 +6584,7 @@ RectTransform:
   - {fileID: 5233312448025626847}
   - {fileID: 5233312447201393292}
   m_Father: {fileID: 0}
-  m_RootOrder: 3
+  m_RootOrder: 4
   m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
   m_AnchorMin: {x: 0.5, y: 0}
   m_AnchorMax: {x: 0.5, y: 0}
@@ -6613,7 +6600,6 @@ GameObject:
   serializedVersion: 6
   m_Component:
   - component: {fileID: 5233312447513285389}
-  - component: {fileID: 5233312447513285388}
   m_Layer: 5
   m_Name: Feedback
   m_TagString: Untagged

Assets/Hangman/Scripts/HangmanGameController.cs

@@ -1,11 +1,13 @@
 using System;
+using System.Collections;
 using System.Collections.Generic;
 using System.Linq;
 using TMPro;
 using UnityEngine;
 using UnityEngine.UI;
+using DigitalRuby.Tween;
 
-public class HangmanGameController : MonoBehaviour
+public class HangmanGameController : AbstractFeedback
 {
     /// <summary>
     /// The scriptable with all the themes, will be used to select a random word for hangman.
@@ -200,11 +202,6 @@ public class HangmanGameController : MonoBehaviour
     /// </summary>
     public GameObject scoreboardEntry;
 
-    /// <summary>
-    /// Accuracy feeback object
-    /// </summary>
-    public Feedback feedback;
-
     /// <summary>
     /// The button to go into the game
     /// </summary>
@@ -215,8 +212,48 @@ public class HangmanGameController : MonoBehaviour
     /// </summary>
     private String currentsign = "";
 
+    /// <summary>
+    /// Reference to the feedback field
+    /// </summary>
+    public TMP_Text feedbackText;
+
+    /// <summary>
+    /// Reference to the progress bar
+    /// </summary>
+    public Slider feedbackProgress;
+
+    /// <summary>
+    /// Reference to the progress bar image, so we can add fancy colors
+    /// </summary>
+    public Image feedbackProgressImage;
+
+    /// <summary>
+    /// Timer to keep track of how long a incorrect sign is performed
+    /// </summary>
+    protected DateTime timer;
+
+    /// <summary>
+    /// Current predicted sign
+    /// </summary>
+    protected string predictedSign = null;
+
+    /// <summary>
+    /// Previous incorrect sign, so we can keep track whether the user is wrong or the user is still changing signs
+    /// </summary>
+    protected string previousIncorrectSign = null;
+
     // Start is called before the first frame update
     void Start()
+    {
+        StartController();
+
+        signPredictor.SetModel(ModelIndex.FINGERSPELLING);
+        AddSelfAsListener();
+    }
+    /// <summary>
+    /// Called at the start of the scene AND when the scene is replayed
+    /// </summary>
+    public void StartController()
     {
         // Make sure the mode starts at zero
         mode = 0;
@@ -240,16 +277,6 @@ public class HangmanGameController : MonoBehaviour
             user.minigames.Add(progress);
         }
         userList.Save();
-
-        // Set calllbacks
-        feedback.getSignCallback = () =>
-        {
-            return "A";
-        };
-        feedback.predictSignCallback = (sign) =>
-        {
-            currentsign = sign;
-        };
     }
 
     /// <summary>
@@ -278,7 +305,7 @@ public class HangmanGameController : MonoBehaviour
         DeleteWord();
         DisplayWord(currentWord);
 
-        replayButton.onClick.AddListener(Start);
+        replayButton.onClick.AddListener(StartController);
         // Call to display the first image, corresponding to a clean image.
         ChangeSprite();
     }
@@ -349,7 +376,7 @@ public class HangmanGameController : MonoBehaviour
     {
         if (mode == 1)
         {
-            if (currentsign != "")
+            if (currentsign != "" && currentsign != null)
             {
                 char letter = currentsign.ToLower()[0];
                 currentsign = "";
@@ -390,7 +417,7 @@ public class HangmanGameController : MonoBehaviour
                 // For the first input char given by the user, check if the letter is in the word that needs to be spelled.
 
                 // Check to make sure the inputfield is not empty
-                if (currentsign != "")
+                if (currentsign != null && currentsign != "")
                 {
                     char firstLetter = currentsign.ToLower()[0];
                     currentsign = "";
@@ -540,21 +567,6 @@ public class HangmanGameController : MonoBehaviour
         }
     }
 
-    /// <summary>
-    /// Randomly shuffle the list of words
-    /// </summary>
-    private void ShuffleWords()
-    {
-        for (int i = words.Length - 1; i > 0; i--)
-        {
-            // Generate a random index between 0 and i (inclusive)
-            int j = UnityEngine.Random.Range(0, i + 1);
-
-            // Swap the values at indices i and j
-            (words[j], words[i]) = (words[i], words[j]);
-        }
-    }
-
     /// <summary>
     /// Update and save the scores
     /// </summary>
@@ -728,4 +740,94 @@ public class HangmanGameController : MonoBehaviour
             rank++;
         }
     }
+
+    protected override IEnumerator UpdateFeedback()
+    {
+        // Get current sign
+        string currentSign = "A";
+        // Get the predicted sign
+        if (signPredictor != null && signPredictor.learnableProbabilities != null &&
+            currentSign != null && signPredictor.learnableProbabilities.ContainsKey(currentSign))
+        {
+            float accuracy = signPredictor.learnableProbabilities[currentSign];
+            if (feedbackText != null && feedbackProgressImage != null)
+            {
+                if (accuracy > 0.90)
+                {
+                    feedbackText.text = "Goed";
+                    feedbackText.color = Color.green;
+                    feedbackProgressImage.color = Color.green;
+                }
+                else if (accuracy > 0.80)
+                {
+                    feedbackText.text = "Bijna...";
+                    Color col = new Color(0xff / 255.0f, 0x66 / 255.0f, 0x00 / 255.0f);
+                    feedbackText.color = col;
+                    feedbackProgressImage.color = col;
+                }
+                else
+                {
+                    feedbackText.text = "Detecteren...";
+                    feedbackText.color = Color.red;
+                    feedbackProgressImage.color = Color.red;
+                }
+
+                float oldValue = feedbackProgress.value;
+                // use an exponential scale 
+                float newValue = Mathf.Exp(4 * (accuracy - 1.0f));
+                feedbackProgress.gameObject.Tween("FeedbackUpdate", oldValue, newValue, 0.2f, TweenScaleFunctions.CubicEaseInOut, (t) =>
+                {
+                    if (feedbackProgress != null)
+                    {
+                        feedbackProgress.value = t.CurrentValue;
+                    }
+                });
+            }
+
+            // Check whether (in)correct sign has high accuracy
+            foreach (var kv in signPredictor.learnableProbabilities)
+            {
+                if (kv.Value > 0.90)
+                {
+                    predictedSign = kv.Key;
+                    // Correct sign
+                    if (predictedSign == currentSign)
+                    {
+                        yield return new WaitForSeconds(1.0f);
+                        CheckEquality(predictedSign);
+                        timer = DateTime.Now;
+                        predictedSign = null;
+                        previousIncorrectSign = null;
+                    }
+                    // Incorrect sign
+                    else
+                    {
+                        if (previousIncorrectSign != predictedSign)
+                        {
+                            timer = DateTime.Now;
+                            previousIncorrectSign = predictedSign;
+                        }
+                        else if (DateTime.Now - timer > TimeSpan.FromSeconds(2.0f))
+                        {
+                            CheckEquality(predictedSign);
+                            timer = DateTime.Now;
+                            predictedSign = null;
+                            previousIncorrectSign = null;
+                        }
+                    }
+                    break;
+                }
+            }
+        }
+        else if (feedbackProgress != null)
+        {
+
+            feedbackProgress.value = 0.0f;
+        }
+
+    }
+    private void CheckEquality(string sign)
+    {
+        currentsign = sign;
+    }
 }

Assets/MediaPipeUnity/Interfaces.meta

@@ -0,0 +1,8 @@
+fileFormatVersion: 2
+guid: 73c615986873dc246893879daf74c05d
+folderAsset: yes
+DefaultImporter:
+  externalObjects: {}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Assets/MediaPipeUnity/Interfaces/Listener.cs

@@ -0,0 +1,8 @@
+using System.Collections;
+using System.Collections.Generic;
+using UnityEngine;
+
+public interface Listener
+{
+    public IEnumerator ProcessIncomingCall();
+}

Assets/MediaPipeUnity/Interfaces/Listener.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: e4c1da9896d9ba2449549a016b5fd15e
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Assets/MediaPipeUnity/Interfaces/SignPredictorInterfaces.asmdef

@@ -0,0 +1,14 @@
+{
+    "name": "SignPredictorInterfaces",
+    "rootNamespace": "",
+    "references": [],
+    "includePlatforms": [],
+    "excludePlatforms": [],
+    "allowUnsafeCode": false,
+    "overrideReferences": false,
+    "precompiledReferences": [],
+    "autoReferenced": true,
+    "defineConstraints": [],
+    "versionDefines": [],
+    "noEngineReferences": false
+}

Assets/MediaPipeUnity/Interfaces/SignPredictorInterfaces.asmdef.meta

@@ -0,0 +1,7 @@
+fileFormatVersion: 2
+guid: f55a02e98b01bc849b30d9650ccd8f15
+AssemblyDefinitionImporter:
+  externalObjects: {}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Assets/MediaPipeUnity/Prefabs/Feedback.prefab

@@ -9,7 +9,6 @@ GameObject:
   serializedVersion: 6
   m_Component:
   - component: {fileID: 4318122119930585316}
-  - component: {fileID: 4318122119930585317}
   m_Layer: 5
   m_Name: Feedback
   m_TagString: Untagged
@@ -39,22 +38,6 @@ RectTransform:
   m_AnchoredPosition: {x: 0, y: 200}
   m_SizeDelta: {x: 500, y: 150}
   m_Pivot: {x: 0.5, y: 0}
---- !u!114 &4318122119930585317
-MonoBehaviour:
-  m_ObjectHideFlags: 0
-  m_CorrespondingSourceObject: {fileID: 0}
-  m_PrefabInstance: {fileID: 0}
-  m_PrefabAsset: {fileID: 0}
-  m_GameObject: {fileID: 4318122119930585319}
-  m_Enabled: 1
-  m_EditorHideFlags: 0
-  m_Script: {fileID: 11500000, guid: 44e682a32ee15cc489bf50f3a06f717b, type: 3}
-  m_Name: 
-  m_EditorClassIdentifier: 
-  feedbackText: {fileID: 4318122120222767928}
-  feedbackProgress: {fileID: 4318122119968934242}
-  feedbackProgressImage: {fileID: 4318122120334233319}
-  signPredictor: {fileID: 0}
 --- !u!1 &4318122119968934244
 GameObject:
   m_ObjectHideFlags: 0

Assets/MediaPipeUnity/ScriptableObjects.meta

@@ -0,0 +1,8 @@
+fileFormatVersion: 2
+guid: 438a3ced42dd6fc4ab38e3a16c1e43a7
+folderAsset: yes
+DefaultImporter:
+  externalObjects: {}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Assets/MediaPipeUnity/ScriptableObjects/ModelList.asset

@@ -0,0 +1,20 @@
+%YAML 1.1
+%TAG !u! tag:unity3d.com,2011:
+--- !u!114 &11400000
+MonoBehaviour:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInstance: {fileID: 0}
+  m_PrefabAsset: {fileID: 0}
+  m_GameObject: {fileID: 0}
+  m_Enabled: 1
+  m_EditorHideFlags: 0
+  m_Script: {fileID: 11500000, guid: 78a3f61c93a08c04496c49ffd10b9021, type: 3}
+  m_Name: ModelList
+  m_EditorClassIdentifier: 
+  currentModelIndex: 0
+  models:
+  - index: 0
+    model: {fileID: 5022602860645237092, guid: e6d85df707405ad4f97c23b07227ee99, type: 3}
+  - index: 1
+    model: {fileID: 5022602860645237092, guid: e6d85df707405ad4f97c23b07227ee99, type: 3}

Assets/MediaPipeUnity/ScriptableObjects/ModelList.asset.meta

@@ -0,0 +1,8 @@
+fileFormatVersion: 2
+guid: 39516e4e6e56f0f4f80647d9c4d8034c
+NativeFormatImporter:
+  externalObjects: {}
+  mainObjectFileID: 11400000
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Assets/MediaPipeUnity/Scripts/AbstractFeedback.cs

@@ -0,0 +1,32 @@
+using DigitalRuby.Tween;
+using Mediapipe.Unity.Tutorial;
+using System;
+using System.Collections;
+using TMPro;
+using UnityEngine;
+using UnityEngine.Events;
+using UnityEngine.UI;
+
+/// <summary>
+/// Class to display feedback during a course
+/// </summary>
+public abstract class AbstractFeedback : MonoBehaviour, Listener
+{
+
+    /// <summary>
+    /// Reference to the sign predictor
+    /// </summary>
+    public SignPredictor signPredictor;
+
+    public IEnumerator ProcessIncomingCall()
+    {
+        yield return StartCoroutine(UpdateFeedback());
+    }
+
+    public void AddSelfAsListener()
+    {
+        signPredictor.listeners.Add(this);
+    }
+
+    protected abstract IEnumerator UpdateFeedback();
+}

Assets/MediaPipeUnity/Scripts/AbstractFeedback.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: 7b5ac794337a54143a6e3077483d96c9
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Assets/MediaPipeUnity/Scripts/Feedback.cs

@@ -1,182 +0,0 @@
-using DigitalRuby.Tween;
-using Mediapipe.Unity.Tutorial;
-using System;
-using System.Collections;
-using TMPro;
-using UnityEngine;
-using UnityEngine.Events;
-using UnityEngine.UI;
-
-/// <summary>
-/// Class to display feedback during a course
-/// </summary>
-public class Feedback : MonoBehaviour
-{
-    /// <summary>
-    /// Reference to the feedback field
-    /// </summary>
-    public TMP_Text feedbackText;
-
-    /// <summary>
-    /// Reference to the progress bar
-    /// </summary>
-    public Slider feedbackProgress;
-
-    /// <summary>
-    /// Reference to the progress bar image, so we can add fancy colors
-    /// </summary>
-    public Image feedbackProgressImage;
-
-    /// <summary>
-    /// Reference to the sign predictor
-    /// </summary>
-    public SignPredictor signPredictor;
-
-    /// <summary>
-    /// Callback for getting the correct sign
-    /// </summary>
-    public Func<string> getSignCallback;
-
-    /// <summary>
-    /// Callback to initiate the next sign
-    /// </summary>
-    public UnityAction<string> predictSignCallback;
-
-    /// <summary>
-    /// Timer to keep track of how long a incorrect sign is performed
-    /// </summary>
-    private DateTime timer;
-
-    /// <summary>
-    /// Current predicted sign
-    /// </summary>
-    private string predictedSign = null;
-
-    /// <summary>
-    /// Previous incorrect sign, so we can keep track whether the user is wrong or the user is still changing signs
-    /// </summary>
-    private string previousIncorrectSign = null;
-
-    /// <summary>
-    /// Start is called before the first frame update
-    /// </summary>
-    void Start()
-    {
-        // Start the coroutine to update the scale every 200 milliseconds
-        StartCoroutine(UpdateFeedback());
-    }
-
-    /// <summary>
-    /// UpdateScale updates the progress bar every 200ms, updated the feedback text, and progress bar color
-    /// If a high enough accuracy is detected, it will go to the next sign
-    /// </summary>
-    /// <returns></returns>
-    IEnumerator UpdateFeedback()
-    {
-        while (true)
-        {
-            if (getSignCallback != null && predictSignCallback != null)
-            {
-
-                // Get current sign
-                string currentSign = getSignCallback();
-                // Get the predicted sign
-                if (signPredictor != null && signPredictor.learnableProbabilities != null &&
-                    currentSign != null && signPredictor.learnableProbabilities.ContainsKey(currentSign))
-                {    
-                    float accuracy = signPredictor.learnableProbabilities[currentSign];
-                    if (feedbackText != null && feedbackProgressImage != null){
-                        if (accuracy > 0.98)
-                        {
-                            // TODO: fix emojis
-                            feedbackText.text = "✨ Perfect ✨";
-                            Color col = new Color(0xff / 255.0f, 0xcc / 255.0f, 0x00 / 255.0f);
-                            feedbackText.color = col;
-                            feedbackProgressImage.color = col;
-                        }
-                        else if (accuracy > 0.95)
-                        {
-                            feedbackText.text = "Super!";
-                            Color col = new Color(0x00 / 255.0f, 0xff / 255.0f, 0xcc / 255.0f);
-                            feedbackText.color = col;
-                            feedbackProgressImage.color = col;
-                        }
-                        else if (accuracy > 0.90)
-                        {
-                            feedbackText.text = "Goed";
-                            feedbackText.color = Color.green;
-                            feedbackProgressImage.color = Color.green;
-                        }
-                        else if (accuracy > 0.80)
-                        {
-                            feedbackText.text = "Bijna...";
-                            Color col = new Color(0xff / 255.0f, 0x66 / 255.0f, 0x00 / 255.0f);
-                            feedbackText.color = col;
-                            feedbackProgressImage.color = col;
-                        }
-                        else
-                        {
-                            feedbackText.text = "Detecteren...";
-                            feedbackText.color = Color.red;
-                            feedbackProgressImage.color = Color.red;
-                        }
-
-                        float oldValue = feedbackProgress.value;
-                        // use an exponential scale 
-                        float newValue = Mathf.Exp(4 * (accuracy - 1.0f));
-                        feedbackProgress.gameObject.Tween("FeedbackUpdate", oldValue, newValue, 0.2f, TweenScaleFunctions.CubicEaseInOut, (t) =>
-                        {
-                            if (feedbackProgress != null)
-                            {
-                                feedbackProgress.value = t.CurrentValue;
-                            }
-                        });
-                    }
-
-                    // Check whether (in)correct sign has high accuracy
-                    foreach (var kv in signPredictor.learnableProbabilities)
-                    {
-                        if (kv.Value > 0.90)
-                        {
-                            predictedSign = kv.Key;
-                            // Correct sign
-                            if (predictedSign == currentSign)
-                            {
-                                yield return new WaitForSeconds(1.0f);
-                                predictSignCallback(predictedSign);
-                                timer = DateTime.Now;
-                                predictedSign = null;
-                                previousIncorrectSign = null;
-                            }
-                            // Incorrect sign
-                            else
-                            {
-                                if (previousIncorrectSign != predictedSign)
-                                {
-                                    timer = DateTime.Now;
-                                    previousIncorrectSign = predictedSign;
-                                }
-                                else if (DateTime.Now - timer > TimeSpan.FromSeconds(2.0f))
-                                {
-                                    predictSignCallback(predictedSign);
-                                    timer = DateTime.Now;
-                                    predictedSign = null;
-                                    previousIncorrectSign = null;
-                                }
-                            }
-                            break;
-                        }
-                    }
-                }
-                else if(feedbackProgress != null)
-                {
-                    
-                    feedbackProgress.value = 0.0f;
-                }
-            }
-
-            // Wait for 200 milliseconds before updating the scale again
-            yield return new WaitForSeconds(0.2f);
-        }
-    }
-}

Assets/MediaPipeUnity/Scripts/SignPredictor.asmdef

@@ -6,7 +6,9 @@
         "GUID:5c2b5ba89f9e74e418232e154bc5cc7a",
         "GUID:04c4d86a70aa56c55a78c61f1ab1a56d",
         "GUID:edc93f477bb73a743a97d6882ed330b3",
-        "GUID:58e104b97fb3752438ada2902a36dcbf"
+        "GUID:58e104b97fb3752438ada2902a36dcbf",
+        "GUID:7f2d0ee6dd21e1d4eb25b71b7a749d25",
+        "GUID:f55a02e98b01bc849b30d9650ccd8f15"
     ],
     "includePlatforms": [],
     "excludePlatforms": [],

Assets/MediaPipeUnity/Scripts/SignPredictor.cs

@@ -19,9 +19,9 @@ namespace Mediapipe.Unity.Tutorial
     public class SignPredictor : MonoBehaviour
     {
         /// <summary>
-        /// Reference to the model used in the SignPredictor
+        /// ModelList, used to change model using ModelIndex
         /// </summary>
-        public NNModel model;
+        public ModelList modelList;
 
         /// <summary>
         /// Reference to the model info file
@@ -141,6 +141,8 @@ namespace Mediapipe.Unity.Tutorial
         /// </summary>
         private Tensor inputTensor;
 
+        public List<Listener> listeners = new List<Listener>();
+
         /// <summary>
         /// Google Mediapipe setup & run
         /// </summary>
@@ -207,14 +209,21 @@ namespace Mediapipe.Unity.Tutorial
 
 
             keypointManager = new KeypointManager(modelInfoFile);
-
             // check if model exists at path
             //var model = ModelLoader.Load(Resources.Load<NNModel>("Models/Fingerspelling/model_A-L"));
-            worker = model.CreateWorker();
+            worker = modelList.GetCurrentModel().CreateWorker();
 
             StartCoroutine(SignRecognitionCoroutine());
             StartCoroutine(MediapipeCoroutine());
         }
+        /// <summary>
+        /// Called at the start of course/Minigame, will set the model before the start of SIgnPredictor is called.
+        /// </summary>
+        /// <param name="index">The index of the model to be used</param>
+        public void SetModel(ModelIndex index)
+        {
+            this.modelList.SetCurrentModel(index);
+        }
 
         /// <summary>
         /// Coroutine which executes the mediapipe pipeline
@@ -315,6 +324,10 @@ namespace Mediapipe.Unity.Tutorial
                         learnableProbabilities.Add(((char)(i + 65)).ToString(), softmaxedOutput2[i]);
                     }
                     //Debug.Log($"prob = [{learnableProbabilities.Aggregate("   ", (t, kv) => $"{t}{kv.Key}:{kv.Value}   ")}]");
+                    foreach(Listener listener in listeners)
+                    {
+                        yield return listener.ProcessIncomingCall();
+                    }
                 }
                 else
                 {

Assets/SpellingBee/PlayModeTests/SpellingBeePlayModeTests.asmdef

@@ -7,7 +7,8 @@
         "InterfacesScripts",
         "Unity.TextMeshPro",
         "SpellingBeeScripts",
-        "AccountsScripts"
+        "AccountsScripts",
+        "SignPredictor"
     ],
     "includePlatforms": [],
     "excludePlatforms": [],

Assets/SpellingBee/Scenes/Game.unity

@@ -1720,6 +1720,10 @@ PrefabInstance:
       propertyPath: m_LocalEulerAnglesHint.z
       value: 0
       objectReference: {fileID: 0}
+    - target: {fileID: 4318122119930585317, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3}
+      propertyPath: m_Enabled
+      value: 0
+      objectReference: {fileID: 0}
     - target: {fileID: 4318122119930585317, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3}
       propertyPath: signPredictor
       value: 
@@ -1739,15 +1743,37 @@ RectTransform:
   m_CorrespondingSourceObject: {fileID: 4318122119930585316, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3}
   m_PrefabInstance: {fileID: 967164043}
   m_PrefabAsset: {fileID: 0}
---- !u!114 &967164045 stripped
+--- !u!114 &967164046 stripped
 MonoBehaviour:
-  m_CorrespondingSourceObject: {fileID: 4318122119930585317, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3}
+  m_CorrespondingSourceObject: {fileID: 4318122120334233319, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3}
   m_PrefabInstance: {fileID: 967164043}
   m_PrefabAsset: {fileID: 0}
   m_GameObject: {fileID: 0}
   m_Enabled: 1
   m_EditorHideFlags: 0
-  m_Script: {fileID: 11500000, guid: 44e682a32ee15cc489bf50f3a06f717b, type: 3}
+  m_Script: {fileID: 11500000, guid: fe87c0e1cc204ed48ad3b37840f39efc, type: 3}
+  m_Name: 
+  m_EditorClassIdentifier: 
+--- !u!114 &967164047 stripped
+MonoBehaviour:
+  m_CorrespondingSourceObject: {fileID: 4318122119968934242, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3}
+  m_PrefabInstance: {fileID: 967164043}
+  m_PrefabAsset: {fileID: 0}
+  m_GameObject: {fileID: 0}
+  m_Enabled: 1
+  m_EditorHideFlags: 0
+  m_Script: {fileID: 11500000, guid: 67db9e8f0e2ae9c40bc1e2b64352a6b4, type: 3}
+  m_Name: 
+  m_EditorClassIdentifier: 
+--- !u!114 &967164048 stripped
+MonoBehaviour:
+  m_CorrespondingSourceObject: {fileID: 4318122120222767928, guid: 7c71c65ecb5fe0449a8b0d178987f016, type: 3}
+  m_PrefabInstance: {fileID: 967164043}
+  m_PrefabAsset: {fileID: 0}
+  m_GameObject: {fileID: 0}
+  m_Enabled: 1
+  m_EditorHideFlags: 0
+  m_Script: {fileID: 11500000, guid: f4688fdb7df04437aeb418b961361dc5, type: 3}
   m_Name: 
   m_EditorClassIdentifier: 
 --- !u!1 &978093274
@@ -3144,7 +3170,7 @@ MonoBehaviour:
       m_Calls:
       - m_Target: {fileID: 1768150807}
         m_TargetAssemblyTypeName: GameController, SpellingBeeScripts
-        m_MethodName: Start
+        m_MethodName: StartController
         m_Mode: 1
         m_Arguments:
           m_ObjectArgument: {fileID: 0}
@@ -3900,7 +3926,7 @@ MonoBehaviour:
   m_Script: {fileID: 11500000, guid: 043ccd99cf82b3cc9bf2e00956ce2b93, type: 3}
   m_Name: 
   m_EditorClassIdentifier: 
-  model: {fileID: 5022602860645237092, guid: e6d85df707405ad4f97c23b07227ee99, type: 3}
+  modelList: {fileID: 11400000, guid: 39516e4e6e56f0f4f80647d9c4d8034c, type: 2}
   modelInfoFile: {fileID: 4900000, guid: fb8b51022bdcd654a9f29c054832a1b5, type: 3}
   configAsset: {fileID: 4900000, guid: 6288c43cdca97374782dac1ea87aa029, type: 3}
   screen: {fileID: 1743003084}
@@ -4552,6 +4578,10 @@ MonoBehaviour:
   m_Script: {fileID: 11500000, guid: 44fbed5ae228de39b9f727def7578d06, type: 3}
   m_Name: 
   m_EditorClassIdentifier: 
+  feedbackText: {fileID: 967164048}
+  feedbackProgress: {fileID: 967164047}
+  feedbackProgressImage: {fileID: 967164046}
+  signPredictor: {fileID: 1592592444}
   themeList: {fileID: 11400000, guid: a247e2ce790f0f746a3bc521e6ab7d58, type: 2}
   userList: {fileID: 11400000, guid: 072bec636a40f7e4e93b0ac624a3bda2, type: 2}
   minigame: {fileID: 11400000, guid: 8a087d241d652634eb4f6352267ea7dc, type: 2}
@@ -4562,7 +4592,6 @@ MonoBehaviour:
   timerText: {fileID: 1843239267}
   bonusTimeText: {fileID: 1812475780}
   Scoreboard: {fileID: 862382568}
-  feedback: {fileID: 967164045}
   gameEndedPanel: {fileID: 757133117}
 --- !u!1 &1812475780
 GameObject:

Assets/SpellingBee/Scripts/GameController.cs

@@ -5,8 +5,9 @@ using System.Linq;
 using TMPro;
 using UnityEngine;
 using UnityEngine.UI;
+using DigitalRuby.Tween;
 
-public partial class GameController : MonoBehaviour
+public partial class GameController : AbstractFeedback
 {
     /// <summary>
     /// All of the words that can be used in this session
@@ -136,20 +137,53 @@ public partial class GameController : MonoBehaviour
     /// </summary>
     public Transform Scoreboard;
 
-    /// <summary>
-    /// Accuracy feeback object
-    /// </summary>
-    public Feedback feedback;
-
     /// <summary>
     /// Reference to the gameEnded panel, so we can update its display
     /// </summary>
     public GameObject gameEndedPanel;
 
+    /// <summary>
+    /// Reference to the feedback field
+    /// </summary>
+    public TMP_Text feedbackText;
+
+    /// <summary>
+    /// Reference to the progress bar
+    /// </summary>
+    public Slider feedbackProgress;
+
+    /// <summary>
+    /// Reference to the progress bar image, so we can add fancy colors
+    /// </summary>
+    public Image feedbackProgressImage;
+
+    /// <summary>
+    /// Timer to keep track of how long a incorrect sign is performed
+    /// </summary>
+    protected DateTime timer;
+
+    /// <summary>
+    /// Current predicted sign
+    /// </summary>
+    protected string predictedSign = null;
+
+    /// <summary>
+    /// Previous incorrect sign, so we can keep track whether the user is wrong or the user is still changing signs
+    /// </summary>
+    protected string previousIncorrectSign = null;
+
     /// <summary>
     /// Start is called before the first frame update
     /// </summary>
     public void Start()
+    {
+        StartController();
+
+        signPredictor.SetModel(currentTheme.modelIndex);
+        AddSelfAsListener();
+    }
+
+    public void StartController()
     {
         correctLetters = 0;
         incorrectLetters = 0;
@@ -182,29 +216,10 @@ public partial class GameController : MonoBehaviour
         userList.Save();
 
         currentTheme = minigame.themeList.themes[minigame.themeList.currentThemeIndex];
-        feedback.signPredictor.model = currentTheme.model;
+        //feedback.signPredictor.ChangeModel(currentTheme.modelIndex);
         words.AddRange(currentTheme.learnables);
         ShuffleWords();
         NextWord();
-
-        // Set calllbacks
-        feedback.getSignCallback = () =>
-        {
-            if (letterIndex < currentWord.Length)
-            {
-                return currentWord[letterIndex].ToString().ToUpper();
-            }
-            return null;
-        };
-        feedback.predictSignCallback = (sign) =>
-        {
-            bool successful = sign.ToUpper() == currentWord[letterIndex].ToString().ToUpper();
-            if (successful)
-            {
-                AddSeconds(secondsPerLetter);
-            }
-            NextLetter(successful);
-        };
     }
 
     /// <summary>
@@ -453,4 +468,106 @@ public partial class GameController : MonoBehaviour
     {
         yield return new WaitForSecondsRealtime(2);
     }
+
+    protected override IEnumerator UpdateFeedback()
+    {
+        // Get current sign
+        string currentSign = GetSign();
+        // Get the predicted sign
+        if (signPredictor != null && signPredictor.learnableProbabilities != null &&
+            currentSign != null && signPredictor.learnableProbabilities.ContainsKey(currentSign))
+        {
+            float accuracy = signPredictor.learnableProbabilities[currentSign];
+            if (feedbackText != null && feedbackProgressImage != null)
+            {
+                if (accuracy > 0.90)
+                {
+                    feedbackText.text = "Goed";
+                    feedbackText.color = Color.green;
+                    feedbackProgressImage.color = Color.green;
+                }
+                else if (accuracy > 0.80)
+                {
+                    feedbackText.text = "Bijna...";
+                    Color col = new Color(0xff / 255.0f, 0x66 / 255.0f, 0x00 / 255.0f);
+                    feedbackText.color = col;
+                    feedbackProgressImage.color = col;
+                }
+                else
+                {
+                    feedbackText.text = "Detecteren...";
+                    feedbackText.color = Color.red;
+                    feedbackProgressImage.color = Color.red;
+                }
+
+                float oldValue = feedbackProgress.value;
+                // use an exponential scale 
+                float newValue = Mathf.Exp(4 * (accuracy - 1.0f));
+                feedbackProgress.gameObject.Tween("FeedbackUpdate", oldValue, newValue, 0.2f, TweenScaleFunctions.CubicEaseInOut, (t) =>
+                {
+                    if (feedbackProgress != null)
+                    {
+                        feedbackProgress.value = t.CurrentValue;
+                    }
+                });
+            }
+
+            // Check whether (in)correct sign has high accuracy
+            foreach (var kv in signPredictor.learnableProbabilities)
+            {
+                if (kv.Value > 0.90)
+                {
+                    predictedSign = kv.Key;
+                    // Correct sign
+                    if (predictedSign == currentSign)
+                    {
+                        yield return new WaitForSeconds(1.0f);
+                        predictSign(predictedSign);
+                        timer = DateTime.Now;
+                        predictedSign = null;
+                        previousIncorrectSign = null;
+                    }
+                    // Incorrect sign
+                    else
+                    {
+                        if (previousIncorrectSign != predictedSign)
+                        {
+                            timer = DateTime.Now;
+                            previousIncorrectSign = predictedSign;
+                        }
+                        else if (DateTime.Now - timer > TimeSpan.FromSeconds(2.0f))
+                        {
+                            predictSign(predictedSign);
+                            timer = DateTime.Now;
+                            predictedSign = null;
+                            previousIncorrectSign = null;
+                        }
+                    }
+                    break;
+                }
+            }
+        }
+        else if (feedbackProgress != null)
+        {
+
+            feedbackProgress.value = 0.0f;
+        }
+        yield return null;
+    }
+
+    public string GetSign(){
+        if (letterIndex<currentWord.Length){
+            return currentWord[letterIndex].ToString().ToUpper();
+        }
+        return null;
+    }
+
+    public void predictSign(string sign) { 
+        bool successful = sign.ToUpper() == currentWord[letterIndex].ToString().ToUpper();
+        if (successful)
+        {
+            AddSeconds(secondsPerLetter);
+        }
+        NextLetter(successful);
+    }
 }

Assets/SpellingBee/Scripts/SpellingBeeScripts.asmdef

@@ -7,7 +7,8 @@
         "GUID:3444c67d5a3a93e5a95a48906078c372",
         "GUID:d0b6b39a21908f94fbbd9f2c196a9725",
         "GUID:5c2b5ba89f9e74e418232e154bc5cc7a",
-        "GUID:7f2d0ee6dd21e1d4eb25b71b7a749d25"
+        "GUID:7f2d0ee6dd21e1d4eb25b71b7a749d25",
+        "GUID:58e104b97fb3752438ada2902a36dcbf"
     ],
     "includePlatforms": [],
     "excludePlatforms": [],