Merge branch 'New_Model' into 'development'

Added model from A - Z

See merge request wesign/unity-application!54

Assets/Accounts/Scripts/UserList.cs

@@ -43,7 +43,7 @@ public class UserList : ScriptableObject
     /// </summary>
     void OnEnable()
     {
-        PATH = $"{Application.dataPath}/users.json";
+        PATH = $"{Application.persistentDataPath}/users.json";
         Load();
     }
 

Assets/Common/Models/FingerSpelling/model_A-L.onnx.meta

@@ -1,5 +1,5 @@
 fileFormatVersion: 2
-guid: 2594d8a9acfef6c4b88ce2618c4169cb
+guid: e6d85df707405ad4f97c23b07227ee99
 ScriptedImporter:
   internalIDToNameTable: []
   externalObjects: {}

Assets/Common/Models/FingerSpelling/model_A-Z.onnx.meta

@@ -1,16 +0,0 @@
-fileFormatVersion: 2
-guid: e6d85df707405ad4f97c23b07227ee99
-ScriptedImporter:
-  internalIDToNameTable: []
-  externalObjects: {}
-  serializedVersion: 2
-  userData: 
-  assetBundleName: 
-  assetBundleVariant: 
-  script: {fileID: 11500000, guid: 683b6cb6d0a474744822c888b46772c9, type: 3}
-  optimizeModel: 1
-  forceArbitraryBatchSize: 1
-  treatErrorsAsWarnings: 0
-  importMode: 1
-  weightsTypeMode: 0
-  activationTypeMode: 0

Assets/Common/ScriptableObjects/FingerspellingTheme.asset

@@ -15,7 +15,7 @@ MonoBehaviour:
   title: Handalfabet
   description: Van A tot Z
   index: 0
-  model: {fileID: 5022602860645237092, guid: 2594d8a9acfef6c4b88ce2618c4169cb, type: 3}
+  model: {fileID: 5022602860645237092, guid: e6d85df707405ad4f97c23b07227ee99, type: 3}
   learnables:
   - name: A
     image: {fileID: 21300000, guid: 4eb4ef55f866f114dafb722f4bd05c76, type: 3}

Assets/Common/ScriptableObjects/SpellingBasicsTheme.asset

@@ -15,7 +15,7 @@ MonoBehaviour:
   title: Basics
   description: Van welkom tot stop
   index: 10
-  model: {fileID: 5022602860645237092, guid: 2594d8a9acfef6c4b88ce2618c4169cb, type: 3}
+  model: {fileID: 5022602860645237092, guid: e6d85df707405ad4f97c23b07227ee99, type: 3}
   learnables:
   - name: Hallo
     image: {fileID: 21300000, guid: f2a020e24bfa24842bccecf6b36d5b79, type: 3}

Assets/Common/ScriptableObjects/SpellingBuildings.asset

@@ -15,7 +15,7 @@ MonoBehaviour:
   title: In de Stad
   description: Van park tot zwembad
   index: 8
-  model: {fileID: 5022602860645237092, guid: 2594d8a9acfef6c4b88ce2618c4169cb, type: 3}
+  model: {fileID: 5022602860645237092, guid: e6d85df707405ad4f97c23b07227ee99, type: 3}
   learnables:
   - name: Park
     image: {fileID: 21300000, guid: 8ea62bbfad596234c9f6ba49c41b3b08, type: 3}

Assets/Common/ScriptableObjects/SpellingFarm.asset

@@ -15,7 +15,7 @@ MonoBehaviour:
   title: Boerderijdieren
   description: Van konijn tot kip
   index: 16
-  model: {fileID: 5022602860645237092, guid: 2594d8a9acfef6c4b88ce2618c4169cb, type: 3}
+  model: {fileID: 5022602860645237092, guid: e6d85df707405ad4f97c23b07227ee99, type: 3}
   learnables:
   - name: kat
     image: {fileID: 21300000, guid: 73f4e028d9efb644aa23538a749667c5, type: 3}

Assets/Common/ScriptableObjects/SpellingFruit.asset

@@ -15,7 +15,7 @@ MonoBehaviour:
   title: Fruit
   description: Van aardbei tot kers
   index: 13
-  model: {fileID: 5022602860645237092, guid: 2594d8a9acfef6c4b88ce2618c4169cb, type: 3}
+  model: {fileID: 5022602860645237092, guid: e6d85df707405ad4f97c23b07227ee99, type: 3}
   learnables:
   - name: Aardbei
     image: {fileID: 21300000, guid: 1a28fa2c46acbb1408633e4e98061d56, type: 3}

Assets/Common/ScriptableObjects/SpellingGeography.asset

@@ -15,7 +15,7 @@ MonoBehaviour:
   title: Geografie
   description: Van hier tot China
   index: 7
-  model: {fileID: 5022602860645237092, guid: 2594d8a9acfef6c4b88ce2618c4169cb, type: 3}
+  model: {fileID: 5022602860645237092, guid: e6d85df707405ad4f97c23b07227ee99, type: 3}
   learnables:
   - name: China
     image: {fileID: 21300000, guid: a5c1efdb79dcfb54aaf2723f4b4842be, type: 3}

Assets/Common/ScriptableObjects/SpellingHobbies.asset

@@ -15,7 +15,7 @@ MonoBehaviour:
   title: Hobbies
   description: Van sport tot muziek
   index: 11
-  model: {fileID: 5022602860645237092, guid: 2594d8a9acfef6c4b88ce2618c4169cb, type: 3}
+  model: {fileID: 5022602860645237092, guid: e6d85df707405ad4f97c23b07227ee99, type: 3}
   learnables:
   - name: Dansen
     image: {fileID: 21300000, guid: 6d405f607ae817744b49f921f0611088, type: 3}

Assets/Common/ScriptableObjects/SpellingPeople.asset

@@ -15,7 +15,7 @@ MonoBehaviour:
   title: Mensen
   description: Van vrienden tot familie
   index: 12
-  model: {fileID: 5022602860645237092, guid: 2594d8a9acfef6c4b88ce2618c4169cb, type: 3}
+  model: {fileID: 5022602860645237092, guid: e6d85df707405ad4f97c23b07227ee99, type: 3}
   learnables:
   - name: Broer
     image: {fileID: 21300000, guid: daebe503f7767b64683d8eeab617e8bc, type: 3}

Assets/Common/ScriptableObjects/SpellingSports.asset

@@ -15,7 +15,7 @@ MonoBehaviour:
   title: Sporten
   description: Van atletiek tot zeilen
   index: 9
-  model: {fileID: 5022602860645237092, guid: 2594d8a9acfef6c4b88ce2618c4169cb, type: 3}
+  model: {fileID: 5022602860645237092, guid: e6d85df707405ad4f97c23b07227ee99, type: 3}
   learnables:
   - name: Atletiek
     image: {fileID: 21300000, guid: 25a05f23961d2b443972f4aa3c2bec29, type: 3}

Assets/Common/ScriptableObjects/SpellingVeggies.asset

@@ -15,7 +15,7 @@ MonoBehaviour:
   title: Groenten
   description: Van erwt tot komkommer
   index: 14
-  model: {fileID: 5022602860645237092, guid: 2594d8a9acfef6c4b88ce2618c4169cb, type: 3}
+  model: {fileID: 5022602860645237092, guid: e6d85df707405ad4f97c23b07227ee99, type: 3}
   learnables:
   - name: Aardappel
     image: {fileID: 21300000, guid: 2610cdbc24a125f43ada7fed67d8f51b, type: 3}

Assets/Common/ScriptableObjects/SpellingWild.asset

@@ -15,7 +15,7 @@ MonoBehaviour:
   title: Dieren in het wild
   description: Van Leeuw tot olifant
   index: 15
-  model: {fileID: 5022602860645237092, guid: 2594d8a9acfef6c4b88ce2618c4169cb, type: 3}
+  model: {fileID: 5022602860645237092, guid: e6d85df707405ad4f97c23b07227ee99, type: 3}
   learnables:
   - name: Vos
     image: {fileID: 21300000, guid: 9d5771baa6ea6e041b066135d6798e1c, type: 3}

Assets/Courses/Scenes/TemplateCourse.unity

@@ -490,7 +490,7 @@ MonoBehaviour:
   m_Script: {fileID: 11500000, guid: 6b3f784c065813a4a8364b1299284816, type: 3}
   m_Name: 
   m_EditorClassIdentifier: 
-  previewModel: {fileID: 5022602860645237092, guid: 2594d8a9acfef6c4b88ce2618c4169cb, type: 3}
+  previewModel: {fileID: 5022602860645237092, guid: e6d85df707405ad4f97c23b07227ee99, type: 3}
   feedbackProgressBar: {fileID: 4318122121437849759}
   previewMessage: {fileID: 2070775951}
   player: {fileID: 993952931}
@@ -1299,7 +1299,7 @@ MonoBehaviour:
   m_Script: {fileID: 11500000, guid: 043ccd99cf82b3cc9bf2e00956ce2b93, type: 3}
   m_Name: 
   m_EditorClassIdentifier: 
-  model: {fileID: 0}
+  model: {fileID: 5022602860645237092, guid: e6d85df707405ad4f97c23b07227ee99, type: 3}
   modelInfoFile: {fileID: 4900000, guid: fb8b51022bdcd654a9f29c054832a1b5, type: 3}
   configAsset: {fileID: 4900000, guid: 6288c43cdca97374782dac1ea87aa029, type: 3}
   screen: {fileID: 378145456}

Assets/MediaPipeUnity/Common/Scripts/KeypointManager.cs

@@ -53,6 +53,107 @@ public class KeypointManager
         return (normalized_x, normalized_y);
     }
 
+    private (List<float>, List<float>) NormalizeHandBohaecek(List<float> hand_x, List<float> hand_y)
+    {
+
+        float min_x = hand_x.Min();
+        float min_y = hand_y.Min();
+
+        float max_x = hand_x.Max();
+        float max_y = hand_y.Max();
+
+        float width = max_x - min_x;
+        float height = max_y - min_y;
+
+        float delta_x = 0;
+        float delta_y = 0;
+
+        if (width == 0 || height == 0)
+        {
+            return (hand_x, hand_y);
+        }
+
+
+        if (width > height){
+            delta_x = ((float)0.1)*width;
+            delta_y = delta_x + ((width - height)/2);
+        }else{
+            delta_y = ((float)0.1)*height;
+            delta_x = delta_y + ((height - width)/2);
+        }
+
+        float starting_x = min_x - delta_x;
+        float starting_y = min_y - delta_y;
+
+        float ending_x = max_x + delta_x;
+        float ending_y = max_y + delta_y;
+
+        float bbox_center_x = (starting_x + ending_x) / 2;
+        float bbox_center_y = (starting_y + ending_y) / 2;
+
+        float bbox_width = ending_x - starting_x;
+        float bbox_height = ending_y - starting_y;
+
+        List<float> normalized_x = new List<float>();
+        List<float> normalized_y = new List<float>();
+
+        for (int i = 0; i < hand_x.Count; i++)
+        {
+            normalized_x.Add((hand_x[i] - bbox_center_x) / bbox_width);
+            normalized_y.Add((hand_y[i] - bbox_center_y) / bbox_height);
+        }
+
+        return (normalized_x, normalized_y);
+    }
+
+    public (List<float>, List<float>) PoseNormalization(List<float> pose_x, List<float> pose_y)
+    {
+
+        float bbox_size = ((float)4.0);
+
+        float shoulder_left_x = pose_x[6];
+        float shoulder_left_y = pose_y[6];
+
+        float shoulder_right_x = pose_x[7];
+        float shoulder_right_y = pose_y[7];
+
+        float shoulder_distance = Mathf.Sqrt(Mathf.Pow(shoulder_left_x - shoulder_right_x, 2) + Mathf.Pow(shoulder_left_y - shoulder_right_y, 2));
+        float shoulder_center_x = (shoulder_left_x + shoulder_right_x) / 2;
+        float shoulder_center_y = (shoulder_left_y + shoulder_right_y) / 2;
+
+        float eye_left_x = pose_x[1];
+        float eye_left_y = pose_y[1];
+
+        float starting_x = shoulder_center_x - (bbox_size/2) * shoulder_distance;
+        float starting_y = eye_left_y - shoulder_distance/2;
+
+        float ending_x = shoulder_center_x + (bbox_size/2) * shoulder_distance;
+        float ending_y = starting_y + (bbox_size - ((float)0.5)) * shoulder_distance;
+
+        float bbox_center_x = (starting_x + ending_x) / 2;
+        float bbox_center_y = (starting_y + ending_y) / 2;
+
+        float bbox_width = ending_x - starting_x;
+        float bbox_height = ending_y - starting_y;
+
+        if (bbox_width == 0 || bbox_height == 0)
+        {
+            return (pose_x, pose_y);
+        }
+
+        List<float> normalized_x = new List<float>();
+        List<float> normalized_y = new List<float>();
+
+        for (int i = 0; i < pose_x.Count; i++)
+        {
+            normalized_x.Add((pose_x[i] - bbox_center_x) / bbox_width);
+            normalized_y.Add((pose_y[i] - bbox_center_y) / bbox_height);
+        }
+
+        return (normalized_x, normalized_y);
+    }
+
+
     public void AddLandmarks(NormalizedLandmarkList poseLandmarks, NormalizedLandmarkList leftHandLandmarks, NormalizedLandmarkList rightHandLandmarks)
     {
         List<float> pose_x = new List<float>();
@@ -105,11 +206,9 @@ public class KeypointManager
             }
         }
 
-        // TODO: Add normalization
-        //Debug.Log($"pose_landMarks = [{modelInfo.pose_landmarks.Aggregate("   ", (t, f) => $"{t}   {f}")}]");
-        //Debug.Log($"hand_landmarks = [{modelInfo.hand_landmarks.Aggregate("   ", (t, f) => $"{t}   {f}")}]");
-        (left_hand_x, left_hand_y) = NormalizeHand(left_hand_x, left_hand_y);
-        (right_hand_x, right_hand_y) = NormalizeHand(right_hand_x, right_hand_y);
+        (pose_x, pose_y) = PoseNormalization(pose_x, pose_y);
+        (left_hand_x, left_hand_y) = NormalizeHandBohaecek(left_hand_x, left_hand_y);
+        (right_hand_x, right_hand_y) = NormalizeHandBohaecek(right_hand_x, right_hand_y);
 
 
         if (keypointsBuffer.Count >= 10)

Assets/Resources.meta

@@ -0,0 +1,8 @@
+fileFormatVersion: 2
+guid: c3f7371598add9549b5351c44a9e17b3
+folderAsset: yes
+DefaultImporter:
+  externalObjects: {}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Assets/Resources/Models.meta

@@ -0,0 +1,8 @@
+fileFormatVersion: 2
+guid: 7992d1284c7de4b089f4155b3e4ada83
+folderAsset: yes
+DefaultImporter:
+  externalObjects: {}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Assets/SpellingBee/Scenes/Game.unity

@@ -3900,7 +3900,7 @@ MonoBehaviour:
   m_Script: {fileID: 11500000, guid: 043ccd99cf82b3cc9bf2e00956ce2b93, type: 3}
   m_Name: 
   m_EditorClassIdentifier: 
-  model: {fileID: 5022602860645237092, guid: 2594d8a9acfef6c4b88ce2618c4169cb, type: 3}
+  model: {fileID: 5022602860645237092, guid: e6d85df707405ad4f97c23b07227ee99, type: 3}
   modelInfoFile: {fileID: 4900000, guid: fb8b51022bdcd654a9f29c054832a1b5, type: 3}
   configAsset: {fileID: 4900000, guid: 6288c43cdca97374782dac1ea87aa029, type: 3}
   screen: {fileID: 1743003084}

Assets/SpellingBee/Scripts/GameController.cs

@@ -227,6 +227,7 @@ public partial class GameController : MonoBehaviour
             int seconds = Mathf.FloorToInt(timerValue % 60.0f);
 
             timerText.text = string.Format("{0:00}:{1:00}", minutes, seconds);
+            
         }
     }
 
@@ -402,9 +403,9 @@ public partial class GameController : MonoBehaviour
         if (wordIndex < words.Count)
         {
             currentWord = words[wordIndex].name;
-            DisplayWord(currentWord);
-
             letterIndex = 0;
+
+            DisplayWord(currentWord);
             wordIndex++;
         }
         else

Assets/users.json.meta

@@ -0,0 +1,7 @@
+fileFormatVersion: 2
+guid: b78a998f611664947a7e5edb192dd6ff
+TextScriptImporter:
+  externalObjects: {}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: