diff --git a/Assets/Common/Interfaces/ModelIndex.cs b/Assets/Common/Interfaces/ModelIndex.cs new file mode 100644 index 0000000..594fd39 --- /dev/null +++ b/Assets/Common/Interfaces/ModelIndex.cs @@ -0,0 +1,11 @@ +using System.Collections; +using System.Collections.Generic; +using UnityEngine; +/// +/// This enum is used to identify each of the SignLanguage models +/// +public enum ModelIndex +{ + FINGERSPELLING, + NONE +} diff --git a/Assets/Common/Interfaces/ModelIndex.cs.meta b/Assets/Common/Interfaces/ModelIndex.cs.meta new file mode 100644 index 0000000..2fdf846 --- /dev/null +++ b/Assets/Common/Interfaces/ModelIndex.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 6dbd5e1100bc81648b52206df369d0a1 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Assets/Common/Interfaces/ModelList.cs b/Assets/Common/Interfaces/ModelList.cs new file mode 100644 index 0000000..d2a4de9 --- /dev/null +++ b/Assets/Common/Interfaces/ModelList.cs @@ -0,0 +1,57 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using UnityEngine; +using Unity.Barracuda; +/// +/// This scriptable will hold tupples of Courseindices and models +/// +[CreateAssetMenu(menuName = "Create new Scriptable/ModelList")] +public class ModelList : ScriptableObject +{ + /// + /// Small class to link a model to a courseIndex irrespective of its position in a list + /// + [Serializable] + public class ModelTuple + { + /// + /// ModelIndex to which the model corresponds + /// + public ModelIndex index; + /// + /// The model itself + /// + public NNModel model; + } + + + /// + /// Index of the currently active model + /// + public int currentModelIndex = 0; + + /// + /// A list of all the models + /// + public List models = new List(); + + /// + /// Get a model by modelindex + /// + /// ModelIndex of the model + /// Model associated with this index, null if no model was found + public NNModel GetCurrentModel() + { + return models.Find(x => x.model == models[currentModelIndex].model)?.model; + } + + /// + /// Function to find a model-index in the list based on its index + /// + /// + public void SetCurrentModel(ModelIndex index) + { + currentModelIndex = models.FindIndex((m) => m.index == index); + } +} diff --git a/Assets/Common/Interfaces/ModelList.cs.meta b/Assets/Common/Interfaces/ModelList.cs.meta new file mode 100644 index 0000000..e31c7ee --- /dev/null +++ b/Assets/Common/Interfaces/ModelList.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 78a3f61c93a08c04496c49ffd10b9021 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Assets/Common/Interfaces/Theme.cs b/Assets/Common/Interfaces/Theme.cs index 94d7e19..838d223 100644 --- a/Assets/Common/Interfaces/Theme.cs +++ b/Assets/Common/Interfaces/Theme.cs @@ -1,5 +1,4 @@ using System.Collections.Generic; -using Unity.Barracuda; using UnityEngine; /// @@ -24,9 +23,9 @@ public class Theme : ScriptableObject public ThemeIndex index; /// - /// Reference to the model used in the SignPredictor + /// The index of the model you want to use /// - public NNModel model; + public ModelIndex modelIndex; /// /// List of all learnable words/letters diff --git a/Assets/Common/ScriptableObjects/AnimalsTheme.asset b/Assets/Common/ScriptableObjects/AnimalsTheme.asset index a7817e0..f2e7a3b 100644 --- a/Assets/Common/ScriptableObjects/AnimalsTheme.asset +++ b/Assets/Common/ScriptableObjects/AnimalsTheme.asset @@ -16,6 +16,7 @@ MonoBehaviour: description: Van vis tot leeuw index: 2 model: {fileID: 0} + modelIndex: 1 learnables: - name: Walvis image: {fileID: 21300000, guid: 2b01165a5836ab14593d7a5862bd6793, type: 3} diff --git a/Assets/Common/ScriptableObjects/ClothesTheme.asset b/Assets/Common/ScriptableObjects/ClothesTheme.asset index bcc40cf..db7d5fa 100644 --- a/Assets/Common/ScriptableObjects/ClothesTheme.asset +++ b/Assets/Common/ScriptableObjects/ClothesTheme.asset @@ -15,6 +15,8 @@ MonoBehaviour: title: Kleren en Kleuren description: Van rok tot sok index: 1 + model: {fileID: 0} + modelIndex: 1 learnables: - name: Blauw image: {fileID: 21300000, guid: 182fb89eba9c64041bef31ca35c4bcd8, type: 3} diff --git a/Assets/Common/ScriptableObjects/FamilyTheme.asset b/Assets/Common/ScriptableObjects/FamilyTheme.asset index 9a804ee..e3c8df7 100644 --- a/Assets/Common/ScriptableObjects/FamilyTheme.asset +++ b/Assets/Common/ScriptableObjects/FamilyTheme.asset @@ -15,6 +15,8 @@ MonoBehaviour: title: Familie description: Van generatie tot generatie index: 6 + model: {fileID: 0} + modelIndex: 1 learnables: - name: Broer image: {fileID: 21300000, guid: eecf67266f150f1489717049489cf16d, type: 3} diff --git a/Assets/Common/ScriptableObjects/FoodTheme.asset b/Assets/Common/ScriptableObjects/FoodTheme.asset index 80a80d8..6d70a39 100644 --- a/Assets/Common/ScriptableObjects/FoodTheme.asset +++ b/Assets/Common/ScriptableObjects/FoodTheme.asset @@ -16,6 +16,7 @@ MonoBehaviour: description: Van kers tot pompoen index: 3 model: {fileID: 0} + modelIndex: 1 learnables: - name: Aardappel image: {fileID: 21300000, guid: 2610cdbc24a125f43ada7fed67d8f51b, type: 3} diff --git a/Assets/Common/ScriptableObjects/HobbyTheme.asset b/Assets/Common/ScriptableObjects/HobbyTheme.asset index c17ad1d..393ac5c 100644 --- a/Assets/Common/ScriptableObjects/HobbyTheme.asset +++ b/Assets/Common/ScriptableObjects/HobbyTheme.asset @@ -15,6 +15,8 @@ MonoBehaviour: title: Hobbies description: Van schilderen tot reizen index: 4 + model: {fileID: 0} + modelIndex: 1 learnables: - name: Dansen image: {fileID: 21300000, guid: 6d405f607ae817744b49f921f0611088, type: 3} diff --git a/Assets/Common/ScriptableObjects/HouseTheme.asset b/Assets/Common/ScriptableObjects/HouseTheme.asset index 3e9bf20..65b7f0e 100644 --- a/Assets/Common/ScriptableObjects/HouseTheme.asset +++ b/Assets/Common/ScriptableObjects/HouseTheme.asset @@ -15,6 +15,8 @@ MonoBehaviour: title: Huis beschrijven description: Van zetel tot villa index: 5 + model: {fileID: 0} + modelIndex: 1 learnables: - name: Keuken image: {fileID: 21300000, guid: b17ce5bf59092b847b084d3400e7a1b4, type: 3} diff --git a/Assets/Common/Scripts/CourseActivityScreen.cs b/Assets/Common/Scripts/CourseActivityScreen.cs index ab0cdb4..40302fa 100644 --- a/Assets/Common/Scripts/CourseActivityScreen.cs +++ b/Assets/Common/Scripts/CourseActivityScreen.cs @@ -56,8 +56,8 @@ public class CourseActivityScreen : MonoBehaviour Course course = courseList.courses[index]; // vvv TEMPORARY STUFF vvv - playButton.SetActive(course.theme.model != null); - previewButton.SetActive(course.theme.model == null); + playButton.SetActive(course.theme.modelIndex != ModelIndex.NONE); + previewButton.SetActive(course.theme.modelIndex == ModelIndex.NONE); // ^^^ TEMPORARY STUFF ^^^ title.text = course.title; diff --git a/Assets/Common/Tests/CommonTests.asmdef b/Assets/Common/Tests/CommonTests.asmdef index aaa7ab0..7e91147 100644 --- a/Assets/Common/Tests/CommonTests.asmdef +++ b/Assets/Common/Tests/CommonTests.asmdef @@ -5,7 +5,9 @@ "UnityEngine.TestRunner", "UnityEditor.TestRunner", "CommonScripts", - "InterfacesScripts" + "InterfacesScripts", + "Unity.Barracuda", + "SignPredictor" ], "includePlatforms": [ "Editor" diff --git a/Assets/Common/Tests/ModelListTest.cs b/Assets/Common/Tests/ModelListTest.cs new file mode 100644 index 0000000..8ea900c --- /dev/null +++ b/Assets/Common/Tests/ModelListTest.cs @@ -0,0 +1,80 @@ +using NUnit.Framework; +using Unity.Barracuda; +using UnityEngine; +/// +/// Test the ModelList class +/// +[TestFixture] +public class ModelListTest +{ + private ModelList modelList; + + /// + /// Setup a ModelList with all possible Models in the enum + /// + [SetUp] + public void Setup_Model() + { + modelList = ScriptableObject.CreateInstance(); + + // Add a Model for each index in the enum + + // Dumb way to access each index in the enum, couldn't find a different way to do it though + foreach (var field in typeof(ModelIndex).GetFields()) + { + if (field.IsLiteral) + { + ModelIndex value = (ModelIndex)field.GetValue(null); + string name = field.Name; + ModelList.ModelTuple model = new ModelList.ModelTuple(); + // This is all we will need to distinguish + model.index = value; + + // Insert in front to guarantee that ModelIndex will not line up with listIndex + modelList.models.Insert(0, model); + } + } + } + /// + /// Check if current model can be correctly gotten as current via GetCurrentModel + /// + [Test] + public void TestGetCurrentModel() + { + System.Random random = new System.Random(); + ModelIndex value = (ModelIndex)random.Next(modelList.models.Count); + modelList.SetCurrentModel(value); + + Assert.AreEqual(modelList.models[modelList.currentModelIndex].model, modelList.GetCurrentModel()); + + // Check if empty model fails gracefully (returns null) + Assert.IsNull(ScriptableObject.CreateInstance().GetCurrentModel()); + } + + /// + /// Check if all models can be correctly set as current via SetCurrentModel + /// + [Test] + public void TestSetCurrentModel() + { + foreach (var field in typeof(ModelIndex).GetFields()) + { + if (field.IsLiteral) + { + ModelIndex value = (ModelIndex)field.GetValue(null); + string name = field.Name; + modelList.SetCurrentModel(value); + + // Fetch the current model and check if its name is the same as the one we made into the current one + ModelList.ModelTuple m = modelList.models[modelList.currentModelIndex]; + + Assert.AreEqual(m.index, value); + Assert.IsTrue(m.model is NNModel || m.model is null); + } + } + ModelList emptyList = ScriptableObject.CreateInstance(); + emptyList.SetCurrentModel(ModelIndex.FINGERSPELLING); + + Assert.IsTrue(emptyList.currentModelIndex == -1); + } +} diff --git a/Assets/Common/Tests/ModelListTest.cs.meta b/Assets/Common/Tests/ModelListTest.cs.meta new file mode 100644 index 0000000..85cdfd3 --- /dev/null +++ b/Assets/Common/Tests/ModelListTest.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 1313e0cc80244354eb6e2d0c1e891941 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Assets/Courses/Scenes/TemplateCourse.unity b/Assets/Courses/Scenes/TemplateCourse.unity index 036592f..b64e582 100644 --- a/Assets/Courses/Scenes/TemplateCourse.unity +++ b/Assets/Courses/Scenes/TemplateCourse.unity @@ -1301,6 +1301,7 @@ MonoBehaviour: m_Script: {fileID: 11500000, guid: 043ccd99cf82b3cc9bf2e00956ce2b93, type: 3} m_Name: m_EditorClassIdentifier: + modelList: {fileID: 11400000, guid: 39516e4e6e56f0f4f80647d9c4d8034c, type: 2} model: {fileID: 5022602860645237092, guid: e6d85df707405ad4f97c23b07227ee99, type: 3} modelInfoFile: {fileID: 4900000, guid: fb8b51022bdcd654a9f29c054832a1b5, type: 3} configAsset: {fileID: 4900000, guid: 6288c43cdca97374782dac1ea87aa029, type: 3} diff --git a/Assets/Courses/Scripts/TemplateCourse.cs b/Assets/Courses/Scripts/TemplateCourse.cs index 6e25232..19b417a 100644 --- a/Assets/Courses/Scripts/TemplateCourse.cs +++ b/Assets/Courses/Scripts/TemplateCourse.cs @@ -128,13 +128,14 @@ public class TemplateCourse : MonoBehaviour { // Setting up course course = courselist.courses[courselist.currentCourseIndex]; - feedback.signPredictor.model = course.theme.model; + feedback.signPredictor.ChangeModel(course.theme.modelIndex); maxWords = course.theme.learnables.Count; // vvv TEMPORARY STUFF vvv - feedbackProgressBar.SetActive(course.theme.model != null); - previewMessage.SetActive(course.theme.model == null); - feedback.signPredictor.model = previewModel; + feedbackProgressBar.SetActive(course.theme.modelIndex != ModelIndex.NONE); + previewMessage.SetActive(course.theme.modelIndex == ModelIndex.NONE); + // Instead, the NONE-modelIndex points to Fingerspelling, which gives the same result + //feedback.signPredictor.model = previewModel; // ^^^ TEMPORARY STUFF ^^^ // Create entry in current user for keeping track of progress diff --git a/Assets/Hangman/Scenes/Hangman.unity b/Assets/Hangman/Scenes/Hangman.unity index 9209fad..073a6fb 100644 --- a/Assets/Hangman/Scenes/Hangman.unity +++ b/Assets/Hangman/Scenes/Hangman.unity @@ -6245,6 +6245,7 @@ MonoBehaviour: m_Script: {fileID: 11500000, guid: 043ccd99cf82b3cc9bf2e00956ce2b93, type: 3} m_Name: m_EditorClassIdentifier: + modelList: {fileID: 11400000, guid: 39516e4e6e56f0f4f80647d9c4d8034c, type: 2} model: {fileID: 5022602860645237092, guid: e6d85df707405ad4f97c23b07227ee99, type: 3} modelInfoFile: {fileID: 4900000, guid: fb8b51022bdcd654a9f29c054832a1b5, type: 3} configAsset: {fileID: 4900000, guid: 6288c43cdca97374782dac1ea87aa029, type: 3} diff --git a/Assets/Hangman/Scripts/HangmanGameController.cs b/Assets/Hangman/Scripts/HangmanGameController.cs index 45f9bec..9771fc6 100644 --- a/Assets/Hangman/Scripts/HangmanGameController.cs +++ b/Assets/Hangman/Scripts/HangmanGameController.cs @@ -241,6 +241,9 @@ public class HangmanGameController : MonoBehaviour } userList.Save(); + // Hangman always uses fingerspelling + feedback.signPredictor.ChangeModel(ModelIndex.FINGERSPELLING); + // Set calllbacks feedback.getSignCallback = () => { diff --git a/Assets/MediaPipeUnity/ScriptableObjects.meta b/Assets/MediaPipeUnity/ScriptableObjects.meta new file mode 100644 index 0000000..256bc92 --- /dev/null +++ b/Assets/MediaPipeUnity/ScriptableObjects.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: 438a3ced42dd6fc4ab38e3a16c1e43a7 +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Assets/MediaPipeUnity/ScriptableObjects/ModelList.asset b/Assets/MediaPipeUnity/ScriptableObjects/ModelList.asset new file mode 100644 index 0000000..12d799c --- /dev/null +++ b/Assets/MediaPipeUnity/ScriptableObjects/ModelList.asset @@ -0,0 +1,20 @@ +%YAML 1.1 +%TAG !u! tag:unity3d.com,2011: +--- !u!114 &11400000 +MonoBehaviour: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 0} + m_Enabled: 1 + m_EditorHideFlags: 0 + m_Script: {fileID: 11500000, guid: 78a3f61c93a08c04496c49ffd10b9021, type: 3} + m_Name: ModelList + m_EditorClassIdentifier: + currentModelIndex: 0 + models: + - index: 0 + model: {fileID: 5022602860645237092, guid: e6d85df707405ad4f97c23b07227ee99, type: 3} + - index: 1 + model: {fileID: 5022602860645237092, guid: e6d85df707405ad4f97c23b07227ee99, type: 3} diff --git a/Assets/MediaPipeUnity/ScriptableObjects/ModelList.asset.meta b/Assets/MediaPipeUnity/ScriptableObjects/ModelList.asset.meta new file mode 100644 index 0000000..6b6b05c --- /dev/null +++ b/Assets/MediaPipeUnity/ScriptableObjects/ModelList.asset.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: 39516e4e6e56f0f4f80647d9c4d8034c +NativeFormatImporter: + externalObjects: {} + mainObjectFileID: 11400000 + userData: + assetBundleName: + assetBundleVariant: diff --git a/Assets/MediaPipeUnity/Scripts/SignPredictor.asmdef b/Assets/MediaPipeUnity/Scripts/SignPredictor.asmdef index cc93292..ba94856 100644 --- a/Assets/MediaPipeUnity/Scripts/SignPredictor.asmdef +++ b/Assets/MediaPipeUnity/Scripts/SignPredictor.asmdef @@ -6,7 +6,8 @@ "GUID:5c2b5ba89f9e74e418232e154bc5cc7a", "GUID:04c4d86a70aa56c55a78c61f1ab1a56d", "GUID:edc93f477bb73a743a97d6882ed330b3", - "GUID:58e104b97fb3752438ada2902a36dcbf" + "GUID:58e104b97fb3752438ada2902a36dcbf", + "GUID:7f2d0ee6dd21e1d4eb25b71b7a749d25" ], "includePlatforms": [], "excludePlatforms": [], diff --git a/Assets/MediaPipeUnity/Scripts/SignPredictor.cs b/Assets/MediaPipeUnity/Scripts/SignPredictor.cs index b1d8338..2ec4263 100644 --- a/Assets/MediaPipeUnity/Scripts/SignPredictor.cs +++ b/Assets/MediaPipeUnity/Scripts/SignPredictor.cs @@ -19,9 +19,9 @@ namespace Mediapipe.Unity.Tutorial public class SignPredictor : MonoBehaviour { /// - /// Reference to the model used in the SignPredictor + /// ModelList, used to change model using ModelIndex /// - public NNModel model; + public ModelList modelList; /// /// Reference to the model info file @@ -210,12 +210,22 @@ namespace Mediapipe.Unity.Tutorial // check if model exists at path //var model = ModelLoader.Load(Resources.Load("Models/Fingerspelling/model_A-L")); - worker = model.CreateWorker(); + worker = modelList.GetCurrentModel().CreateWorker(); StartCoroutine(SignRecognitionCoroutine()); StartCoroutine(MediapipeCoroutine()); } + public void ChangeModel(ModelIndex index) + { + this.modelList.SetCurrentModel(index); + // If a worker already existed, we throw it out + worker?.Dispose(); + + // Add a new worker for the new model + worker = modelList.GetCurrentModel().CreateWorker(); + } + /// /// Coroutine which executes the mediapipe pipeline /// diff --git a/Assets/SpellingBee/Scenes/Game.unity b/Assets/SpellingBee/Scenes/Game.unity index 0f7aabc..274be82 100644 --- a/Assets/SpellingBee/Scenes/Game.unity +++ b/Assets/SpellingBee/Scenes/Game.unity @@ -3900,6 +3900,7 @@ MonoBehaviour: m_Script: {fileID: 11500000, guid: 043ccd99cf82b3cc9bf2e00956ce2b93, type: 3} m_Name: m_EditorClassIdentifier: + modelList: {fileID: 11400000, guid: 39516e4e6e56f0f4f80647d9c4d8034c, type: 2} model: {fileID: 5022602860645237092, guid: e6d85df707405ad4f97c23b07227ee99, type: 3} modelInfoFile: {fileID: 4900000, guid: fb8b51022bdcd654a9f29c054832a1b5, type: 3} configAsset: {fileID: 4900000, guid: 6288c43cdca97374782dac1ea87aa029, type: 3} diff --git a/Assets/SpellingBee/Scripts/GameController.cs b/Assets/SpellingBee/Scripts/GameController.cs index 94b5f95..d0605fc 100644 --- a/Assets/SpellingBee/Scripts/GameController.cs +++ b/Assets/SpellingBee/Scripts/GameController.cs @@ -182,7 +182,7 @@ public partial class GameController : MonoBehaviour userList.Save(); currentTheme = minigame.themeList.themes[minigame.themeList.currentThemeIndex]; - feedback.signPredictor.model = currentTheme.model; + feedback.signPredictor.ChangeModel(currentTheme.modelIndex); words.AddRange(currentTheme.learnables); ShuffleWords(); NextWord();