WeSign/unity-application
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 6 Wiki Activity

Resolve WES-100 "Natml integration"

Browse Source
This commit is contained in:
Jelle De Geest
2023-04-03 14:14:49 +00:00
parent edf1805a92
commit f95e34c6fe
425 changed files with 525 additions and 96655 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
Assets/Common/Interfaces/InterfacesScripts.asmdef
View File

@@ -2,7 +2,8 @@
"name": "InterfacesScripts",
"rootNamespace": "",
"references": [
"GUID:5c2b5ba89f9e74e418232e154bc5cc7a"
"GUID:5c2b5ba89f9e74e418232e154bc5cc7a",
"GUID:d23f64cfd3b314bb4a18a8284c99bf5e"
],
"includePlatforms": [],
"excludePlatforms": [],

3
Assets/Common/Interfaces/InterfacesScripts.asmdef.meta
View File

@@ -1,7 +1,6 @@
fileFormatVersion: 2
guid: 7f2d0ee6dd21e1d4eb25b71b7a749d25
folderAsset: yes
DefaultImporter:
AssemblyDefinitionImporter:
externalObjects: {}
userData:
assetBundleName:

4
Assets/Common/Interfaces/ModelIndex.cs
View File

@@ -6,6 +6,6 @@ using UnityEngine;
/// </summary>
public enum ModelIndex
{
FINGERSPELLING,
NONE
NONE,
FINGERSPELLING
}

40
Assets/Common/Interfaces/ModelList.cs
View File

@@ -1,8 +1,7 @@
using NatML;
using System;
using System.Collections;
using System.Collections.Generic;
using UnityEngine;
using Unity.Barracuda;
/// <summary>
/// This scriptable will hold tupples of Courseindices and models
/// </summary>
@@ -22,28 +21,49 @@ public class ModelList : ScriptableObject
/// <summary>
/// The model itself
/// </summary>
public NNModel model;
public MLModelData modelWINDOWS;
/// <summary>
/// The model itself
/// </summary>
public MLModelData modelMAC;
}
/// <summary>
/// Index of the currently active model
/// </summary>
public int currentModelIndex = 0;
/// <summary>
/// A list of all the models
/// </summary>
public List<ModelTuple> models = new List<ModelTuple>();
/// <summary>
/// Index of the currently active model
/// </summary>
public int currentModelIndex = 0;
/// <summary>
/// Get a model by modelindex
/// </summary>
/// <param name="modelIndex">ModelIndex of the model</param>
/// <returns>Model associated with this index, null if no model was found</returns>
public NNModel GetCurrentModel()
public MLModelData GetCurrentModel()
{
return models.Find(x => x.model == models[currentModelIndex].model)?.model;
// Select Model based on OS
#if (UNITY_STANDALONE_WIN || UNITY_EDITOR_WIN)
return models.Find(x => x.modelWINDOWS == models[currentModelIndex].modelWINDOWS)?.modelWINDOWS;
#elif (UNITY_STANDALONE_OSX || UNITY_EDITOR_OSX)
return models.Find(x => x.modelMAC == models[currentModelIndex].modelMAC)?.modelMAC;
#endif
return null;
}
/// <summary>
/// Function to check if the modelIndex has been set
/// </summary>
/// <returns></returns>
public bool HasValidModel()
{
return models[currentModelIndex].index != (int)ModelIndex.NONE;
}
/// <summary>

1
Assets/Common/Interfaces/Theme.cs
View File

@@ -27,6 +27,7 @@ public class Theme : ScriptableObject
/// </summary>
public ModelIndex modelIndex;
/// <summary>
/// List of all learnable words/letters
/// </summary>

8
Assets/Common/Models/FingerSpelling/model_A-L.onnx.meta
View File

@@ -7,10 +7,4 @@ ScriptedImporter:
userData:
assetBundleName:
assetBundleVariant:
script: {fileID: 11500000, guid: 683b6cb6d0a474744822c888b46772c9, type: 3}
optimizeModel: 1
forceArbitraryBatchSize: 1
treatErrorsAsWarnings: 0
importMode: 1
weightsTypeMode: 0
activationTypeMode: 0
script: {fileID: 11500000, guid: 8264490bef67c46f2982e6dd3f5e46cd, type: 3}

BIN
Assets/Common/Models/FingerSpelling/model_A-Z2.onnx Normal file
View File

Binary file not shown.

10
Assets/Common/Models/FingerSpelling/model_A-Z2.onnx.meta Normal file
View File

@@ -0,0 +1,10 @@
fileFormatVersion: 2
guid: fdbf401e965a6bf4a87637cd519f2715
ScriptedImporter:
internalIDToNameTable: []
externalObjects: {}
serializedVersion: 2
userData:
assetBundleName:
assetBundleVariant:
script: {fileID: 11500000, guid: 8264490bef67c46f2982e6dd3f5e46cd, type: 3}

2
Assets/Common/ScriptableObjects/FingerspellingTheme.asset
View File

@@ -15,7 +15,7 @@ MonoBehaviour:
title: Handalfabet
description: Van A tot Z
index: 0
model: {fileID: 5022602860645237092, guid: e6d85df707405ad4f97c23b07227ee99, type: 3}
modelIndex: 1
learnables:
- name: A
image: {fileID: 21300000, guid: 4eb4ef55f866f114dafb722f4bd05c76, type: 3}

4
Assets/Common/Tests/CommonTests.asmdef
View File

@@ -6,8 +6,8 @@
"UnityEditor.TestRunner",
"CommonScripts",
"InterfacesScripts",
"Unity.Barracuda",
"SignPredictor"
"SignPredictor",
"NatML.ML"
],
"includePlatforms": [
"Editor"

14
Assets/Common/Tests/ModelListTest.cs
View File

@@ -1,5 +1,5 @@
using NatML;
using NUnit.Framework;
using Unity.Barracuda;
using UnityEngine;
/// <summary>
/// Test the ModelList class
@@ -45,7 +45,11 @@ public class ModelListTest
ModelIndex value = (ModelIndex)random.Next(modelList.models.Count);
modelList.SetCurrentModel(value);
Assert.AreEqual(modelList.models[modelList.currentModelIndex].model, modelList.GetCurrentModel());
#if (UNITY_STANDALONE_WIN || UNITY_EDITOR_WIN)
Assert.AreEqual(modelList.models[modelList.currentModelIndex].modelWINDOWS, modelList.GetCurrentModel());
#elif (UNITY_STANDALONE_OSX || UNITY_EDITOR_OSX)
Assert.AreEqual(modelList.models[modelList.currentModelIndex].modelMAC, modelList.GetCurrentModel());
#endif
// Check if empty model fails gracefully (returns null)
Assert.IsNull(ScriptableObject.CreateInstance<ModelList>().GetCurrentModel());
@@ -69,7 +73,11 @@ public class ModelListTest
ModelList.ModelTuple m = modelList.models[modelList.currentModelIndex];
Assert.AreEqual(m.index, value);
Assert.IsTrue(m.model is NNModel || m.model is null);
#if (UNITY_STANDALONE_WIN || UNITY_EDITOR_WIN)
Assert.IsTrue(m.modelWINDOWS is MLModelData || m.modelWINDOWS is null);
#elif (UNITY_STANDALONE_OSX || UNITY_EDITOR_OSX)
Assert.IsTrue(m.modelMAC is MLModelData || m.modelMAC is null);
#endif
}
}
ModelList emptyList = ScriptableObject.CreateInstance<ModelList>();

5
Assets/Courses/Scripts/CourseScripts.asmdef
View File

@@ -5,8 +5,9 @@
"Unity.TextMeshPro",
"AccountsScripts",
"InterfacesScripts",
"Tween",
"SignPredictor"
"SignPredictor",
"NatML.ML",
"Tween"
],
"includePlatforms": [],
"excludePlatforms": [],

3
Assets/Courses/Scripts/CoursesController.cs
View File

@@ -152,8 +152,7 @@ public class CoursesController : AbstractFeedback
void Start()
{
StartCourseController();
signPredictor.SetModel(course.theme.modelIndex);
signPredictor.ChangeModel(course.theme.modelIndex);
AddSelfAsListener();
}
/// <summary>

471
Assets/Hangman/Scenes/HangmanGame.unity
View File

@@ -38,7 +38,7 @@ RenderSettings:
m_ReflectionIntensity: 1
m_CustomReflection: {fileID: 0}
m_Sun: {fileID: 0}
m_IndirectSpecularColor: {r: 0.37311918, g: 0.3807398, b: 0.35872716, a: 1}
m_IndirectSpecularColor: {r: 0.37311953, g: 0.38074014, b: 0.3587274, a: 1}
m_UseRadianceAmbientProbe: 0
--- !u!157 &3
LightmapSettings:
@@ -6416,472 +6416,3 @@ CanvasRenderer:
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 2039368310}
m_CullTransparentMesh: 1
--- !u!114 &5233312447201393291
MonoBehaviour:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 5233312447201393293}
m_Enabled: 1
m_EditorHideFlags: 0
m_Script: {fileID: 11500000, guid: 67db9e8f0e2ae9c40bc1e2b64352a6b4, type: 3}
m_Name:
m_EditorClassIdentifier:
m_Navigation:
m_Mode: 3
m_WrapAround: 0
m_SelectOnUp: {fileID: 0}
m_SelectOnDown: {fileID: 0}
m_SelectOnLeft: {fileID: 0}
m_SelectOnRight: {fileID: 0}
m_Transition: 1
m_Colors:
m_NormalColor: {r: 1, g: 1, b: 1, a: 1}
m_HighlightedColor: {r: 0.9607843, g: 0.9607843, b: 0.9607843, a: 1}
m_PressedColor: {r: 0.78431374, g: 0.78431374, b: 0.78431374, a: 1}
m_SelectedColor: {r: 0.9607843, g: 0.9607843, b: 0.9607843, a: 1}
m_DisabledColor: {r: 0.78431374, g: 0.78431374, b: 0.78431374, a: 0.5019608}
m_ColorMultiplier: 1
m_FadeDuration: 0.1
m_SpriteState:
m_HighlightedSprite: {fileID: 0}
m_PressedSprite: {fileID: 0}
m_SelectedSprite: {fileID: 0}
m_DisabledSprite: {fileID: 0}
m_AnimationTriggers:
m_NormalTrigger: Normal
m_HighlightedTrigger: Highlighted
m_PressedTrigger: Pressed
m_SelectedTrigger: Selected
m_DisabledTrigger: Disabled
m_Interactable: 1
m_TargetGraphic: {fileID: 0}
m_FillRect: {fileID: 5233312447919013132}
m_HandleRect: {fileID: 0}
m_Direction: 0
m_MinValue: 0
m_MaxValue: 1
m_WholeNumbers: 0
m_Value: 0
m_OnValueChanged:
m_PersistentCalls:
m_Calls: []
--- !u!224 &5233312447201393292
RectTransform:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 5233312447201393293}
m_LocalRotation: {x: 0, y: 0, z: 0, w: 1}
m_LocalPosition: {x: 0, y: 0, z: 0}
m_LocalScale: {x: 1, y: 1, z: 1}
m_ConstrainProportionsScale: 0
m_Children:
- {fileID: 5233312448534255807}
- {fileID: 5233312448785575104}
m_Father: {fileID: 5233312447513285389}
m_RootOrder: 1
m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
m_AnchorMin: {x: 0, y: 0}
m_AnchorMax: {x: 1, y: 0}
m_AnchoredPosition: {x: 0, y: 0}
m_SizeDelta: {x: 0, y: 50}
m_Pivot: {x: 0.5, y: 0}
--- !u!1 &5233312447201393293
GameObject:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
serializedVersion: 6
m_Component:
- component: {fileID: 5233312447201393292}
- component: {fileID: 5233312447201393291}
m_Layer: 5
m_Name: Progress
m_TagString: Untagged
m_Icon: {fileID: 0}
m_NavMeshLayer: 0
m_StaticEditorFlags: 0
m_IsActive: 1
--- !u!114 &5233312447513285388
MonoBehaviour:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 5233312447513285390}
m_Enabled: 1
m_EditorHideFlags: 0
m_Script: {fileID: 11500000, guid: 44e682a32ee15cc489bf50f3a06f717b, type: 3}
m_Name:
m_EditorClassIdentifier:
feedbackText: {fileID: 0}
feedbackProgress: {fileID: 0}
feedbackProgressImage: {fileID: 0}
signPredictor: {fileID: 1991376311}
--- !u!224 &5233312447513285389
RectTransform:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 5233312447513285390}
m_LocalRotation: {x: -0, y: -0, z: -0, w: 1}
m_LocalPosition: {x: 0, y: 0, z: 0}
m_LocalScale: {x: 1, y: 1, z: 1}
m_ConstrainProportionsScale: 0
m_Children:
- {fileID: 5233312448025626847}
- {fileID: 5233312447201393292}
m_Father: {fileID: 0}
m_RootOrder: 5
m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
m_AnchorMin: {x: 0.5, y: 0}
m_AnchorMax: {x: 0.5, y: 0}
m_AnchoredPosition: {x: 960, y: 200}
m_SizeDelta: {x: 500, y: 150}
m_Pivot: {x: 0.5, y: 0}
--- !u!1 &5233312447513285390
GameObject:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
serializedVersion: 6
m_Component:
- component: {fileID: 5233312447513285389}
- component: {fileID: 5233312447513285388}
m_Layer: 5
m_Name: Feedback
m_TagString: Untagged
m_Icon: {fileID: 0}
m_NavMeshLayer: 0
m_StaticEditorFlags: 0
m_IsActive: 1
--- !u!224 &5233312447919013132
RectTransform:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 5233312447919013135}
m_LocalRotation: {x: -0, y: -0, z: -0, w: 1}
m_LocalPosition: {x: 0, y: 0, z: 0}
m_LocalScale: {x: 1, y: 1, z: 1}
m_ConstrainProportionsScale: 0
m_Children: []
m_Father: {fileID: 5233312448785575104}
m_RootOrder: 0
m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
m_AnchorMin: {x: 0, y: 0}
m_AnchorMax: {x: 0, y: 0}
m_AnchoredPosition: {x: 0, y: 0}
m_SizeDelta: {x: 10, y: 0}
m_Pivot: {x: 0.5, y: 0.5}
--- !u!222 &5233312447919013133
CanvasRenderer:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 5233312447919013135}
m_CullTransparentMesh: 1
--- !u!114 &5233312447919013134
MonoBehaviour:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 5233312447919013135}
m_Enabled: 1
m_EditorHideFlags: 0
m_Script: {fileID: 11500000, guid: fe87c0e1cc204ed48ad3b37840f39efc, type: 3}
m_Name:
m_EditorClassIdentifier:
m_Material: {fileID: 0}
m_Color: {r: 1, g: 0, b: 0, a: 1}
m_RaycastTarget: 1
m_RaycastPadding: {x: 0, y: 0, z: 0, w: 0}
m_Maskable: 1
m_OnCullStateChanged:
m_PersistentCalls:
m_Calls: []
m_Sprite: {fileID: 10905, guid: 0000000000000000f000000000000000, type: 0}
m_Type: 1
m_PreserveAspect: 0
m_FillCenter: 1
m_FillMethod: 4
m_FillAmount: 1
m_FillClockwise: 1
m_FillOrigin: 0
m_UseSpriteMesh: 0
m_PixelsPerUnitMultiplier: 1
--- !u!1 &5233312447919013135
GameObject:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
serializedVersion: 6
m_Component:
- component: {fileID: 5233312447919013132}
- component: {fileID: 5233312447919013133}
- component: {fileID: 5233312447919013134}
m_Layer: 5
m_Name: Fill
m_TagString: Untagged
m_Icon: {fileID: 0}
m_NavMeshLayer: 0
m_StaticEditorFlags: 0
m_IsActive: 1
--- !u!222 &5233312448025626832
CanvasRenderer:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 5233312448025626834}
m_CullTransparentMesh: 1
--- !u!114 &5233312448025626833
MonoBehaviour:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 5233312448025626834}
m_Enabled: 1
m_EditorHideFlags: 0
m_Script: {fileID: 11500000, guid: f4688fdb7df04437aeb418b961361dc5, type: 3}
m_Name:
m_EditorClassIdentifier:
m_Material: {fileID: 0}
m_Color: {r: 1, g: 1, b: 1, a: 1}
m_RaycastTarget: 1
m_RaycastPadding: {x: 0, y: 0, z: 0, w: 0}
m_Maskable: 1
m_OnCullStateChanged:
m_PersistentCalls:
m_Calls: []
m_text: Detecteren ...
m_isRightToLeft: 0
m_fontAsset: {fileID: 11400000, guid: 8f586378b4e144a9851e7b34d9b748ee, type: 2}
m_sharedMaterial: {fileID: 2180264, guid: 8f586378b4e144a9851e7b34d9b748ee, type: 2}
m_fontSharedMaterials: []
m_fontMaterial: {fileID: 0}
m_fontMaterials: []
m_fontColor32:
serializedVersion: 2
rgba: 4282188031
m_fontColor: {r: 0.5803922, g: 0.58431375, b: 0.6, a: 1}
m_enableVertexGradient: 0
m_colorMode: 3
m_fontColorGradient:
topLeft: {r: 1, g: 1, b: 1, a: 1}
topRight: {r: 1, g: 1, b: 1, a: 1}
bottomLeft: {r: 1, g: 1, b: 1, a: 1}
bottomRight: {r: 1, g: 1, b: 1, a: 1}
m_fontColorGradientPreset: {fileID: 0}
m_spriteAsset: {fileID: 0}
m_tintAllSprites: 0
m_StyleSheet: {fileID: 0}
m_TextStyleHashCode: -1183493901
m_overrideHtmlColors: 0
m_faceColor:
serializedVersion: 2
rgba: 4294967295
m_fontSize: 48
m_fontSizeBase: 48
m_fontWeight: 400
m_enableAutoSizing: 0
m_fontSizeMin: 18
m_fontSizeMax: 72
m_fontStyle: 1
m_HorizontalAlignment: 2
m_VerticalAlignment: 512
m_textAlignment: 65535
m_characterSpacing: 0
m_wordSpacing: 0
m_lineSpacing: 0
m_lineSpacingMax: 0
m_paragraphSpacing: 0
m_charWidthMaxAdj: 0
m_enableWordWrapping: 1
m_wordWrappingRatios: 0.4
m_overflowMode: 0
m_linkedTextComponent: {fileID: 0}
parentLinkedComponent: {fileID: 0}
m_enableKerning: 1
m_enableExtraPadding: 0
checkPaddingRequired: 0
m_isRichText: 1
m_parseCtrlCharacters: 1
m_isOrthographic: 1
m_isCullingEnabled: 0
m_horizontalMapping: 0
m_verticalMapping: 0
m_uvLineOffset: 0
m_geometrySortingOrder: 0
m_IsTextObjectScaleStatic: 0
m_VertexBufferAutoSizeReduction: 0
m_useMaxVisibleDescender: 1
m_pageToDisplay: 1
m_margin: {x: 0, y: 0, z: 0, w: 0}
m_isUsingLegacyAnimationComponent: 0
m_isVolumetricText: 0
m_hasFontAssetChanged: 0
m_baseMaterial: {fileID: 0}
m_maskOffset: {x: 0, y: 0, z: 0, w: 0}
--- !u!1 &5233312448025626834
GameObject:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
serializedVersion: 6
m_Component:
- component: {fileID: 5233312448025626847}
- component: {fileID: 5233312448025626832}
- component: {fileID: 5233312448025626833}
m_Layer: 5
m_Name: Text
m_TagString: Untagged
m_Icon: {fileID: 0}
m_NavMeshLayer: 0
m_StaticEditorFlags: 0
m_IsActive: 1
--- !u!224 &5233312448025626847
RectTransform:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 5233312448025626834}
m_LocalRotation: {x: -0, y: -0, z: -0, w: 1}
m_LocalPosition: {x: 0, y: 0, z: 0}
m_LocalScale: {x: 1, y: 1, z: 1}
m_ConstrainProportionsScale: 0
m_Children: []
m_Father: {fileID: 5233312447513285389}
m_RootOrder: 0
m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
m_AnchorMin: {x: 0.5, y: 1}
m_AnchorMax: {x: 0.5, y: 1}
m_AnchoredPosition: {x: 0, y: 0}
m_SizeDelta: {x: 500, y: 100}
m_Pivot: {x: 0.5, y: 1}
--- !u!1 &5233312448534255792
GameObject:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
serializedVersion: 6
m_Component:
- component: {fileID: 5233312448534255807}
- component: {fileID: 5233312448534255805}
- component: {fileID: 5233312448534255806}
m_Layer: 5
m_Name: Background
m_TagString: Untagged
m_Icon: {fileID: 0}
m_NavMeshLayer: 0
m_StaticEditorFlags: 0
m_IsActive: 1
--- !u!222 &5233312448534255805
CanvasRenderer:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 5233312448534255792}
m_CullTransparentMesh: 1
--- !u!114 &5233312448534255806
MonoBehaviour:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 5233312448534255792}
m_Enabled: 1
m_EditorHideFlags: 0
m_Script: {fileID: 11500000, guid: fe87c0e1cc204ed48ad3b37840f39efc, type: 3}
m_Name:
m_EditorClassIdentifier:
m_Material: {fileID: 0}
m_Color: {r: 1, g: 1, b: 1, a: 1}
m_RaycastTarget: 1
m_RaycastPadding: {x: 0, y: 0, z: 0, w: 0}
m_Maskable: 1
m_OnCullStateChanged:
m_PersistentCalls:
m_Calls: []
m_Sprite: {fileID: 10907, guid: 0000000000000000f000000000000000, type: 0}
m_Type: 1
m_PreserveAspect: 0
m_FillCenter: 1
m_FillMethod: 4
m_FillAmount: 1
m_FillClockwise: 1
m_FillOrigin: 0
m_UseSpriteMesh: 0
m_PixelsPerUnitMultiplier: 1
--- !u!224 &5233312448534255807
RectTransform:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 5233312448534255792}
m_LocalRotation: {x: -0, y: -0, z: -0, w: 1}
m_LocalPosition: {x: 0, y: 0, z: 0}
m_LocalScale: {x: 1, y: 1, z: 1}
m_ConstrainProportionsScale: 0
m_Children: []
m_Father: {fileID: 5233312447201393292}
m_RootOrder: 0
m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
m_AnchorMin: {x: 0, y: 0}
m_AnchorMax: {x: 1, y: 1}
m_AnchoredPosition: {x: 0, y: 0}
m_SizeDelta: {x: 0, y: 0}
m_Pivot: {x: 0.5, y: 0.5}
--- !u!224 &5233312448785575104
RectTransform:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 5233312448785575105}
m_LocalRotation: {x: -0, y: -0, z: -0, w: 1}
m_LocalPosition: {x: 0, y: 0, z: 0}
m_LocalScale: {x: 1, y: 1, z: 1}
m_ConstrainProportionsScale: 0
m_Children:
- {fileID: 5233312447919013132}
m_Father: {fileID: 5233312447201393292}
m_RootOrder: 1
m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
m_AnchorMin: {x: 0, y: 0}
m_AnchorMax: {x: 1, y: 1}
m_AnchoredPosition: {x: 0, y: 0}
m_SizeDelta: {x: 0, y: 0}
m_Pivot: {x: 0.5, y: 0.5}
--- !u!1 &5233312448785575105
GameObject:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
serializedVersion: 6
m_Component:
- component: {fileID: 5233312448785575104}
m_Layer: 5
m_Name: Fill Area
m_TagString: Untagged
m_Icon: {fileID: 0}
m_NavMeshLayer: 0
m_StaticEditorFlags: 0
m_IsActive: 1

2
Assets/Hangman/Scripts/HangmanController.cs
View File

@@ -244,7 +244,7 @@ public class HangmanController : AbstractFeedback
{
StartController();
signPredictor.SetModel(ModelIndex.FINGERSPELLING);
signPredictor.ChangeModel(ModelIndex.FINGERSPELLING);
AddSelfAsListener();
}
/// <summary>

21
Assets/MediaPipeUnity/Common/Scripts/KeypointManager.cs
View File

@@ -74,12 +74,15 @@ public class KeypointManager
}
if (width > height){
delta_x = ((float)0.1)*width;
delta_y = delta_x + ((width - height)/2);
}else{
delta_y = ((float)0.1)*height;
delta_x = delta_y + ((height - width)/2);
if (width > height)
{
delta_x = ((float)0.1) * width;
delta_y = delta_x + ((width - height) / 2);
}
else
{
delta_y = ((float)0.1) * height;
delta_x = delta_y + ((height - width) / 2);
}
float starting_x = min_x - delta_x;
@@ -124,10 +127,10 @@ public class KeypointManager
float eye_left_x = pose_x[1];
float eye_left_y = pose_y[1];
float starting_x = shoulder_center_x - (bbox_size/2) * shoulder_distance;
float starting_y = eye_left_y - shoulder_distance/2;
float starting_x = shoulder_center_x - (bbox_size / 2) * shoulder_distance;
float starting_y = eye_left_y - shoulder_distance / 2;
float ending_x = shoulder_center_x + (bbox_size/2) * shoulder_distance;
float ending_x = shoulder_center_x + (bbox_size / 2) * shoulder_distance;
float ending_y = starting_y + (bbox_size - ((float)0.5)) * shoulder_distance;
float bbox_center_x = (starting_x + ending_x) / 2;

6
Assets/MediaPipeUnity/ScriptableObjects/ModelList.asset
View File

@@ -15,6 +15,8 @@ MonoBehaviour:
currentModelIndex: 0
models:
- index: 0
model: {fileID: 5022602860645237092, guid: e6d85df707405ad4f97c23b07227ee99, type: 3}
modelWINDOWS: {fileID: 0}
modelMAC: {fileID: 0}
- index: 1
model: {fileID: 0}
modelWINDOWS: {fileID: 8538825877217656561, guid: fdbf401e965a6bf4a87637cd519f2715, type: 3}
modelMAC: {fileID: 0}

6
Assets/MediaPipeUnity/Scripts/AbstractFeedback.cs
View File

@@ -1,11 +1,5 @@
using DigitalRuby.Tween;
using Mediapipe.Unity.Tutorial;
using System;
using System.Collections;
using TMPro;
using UnityEngine;
using UnityEngine.Events;
using UnityEngine.UI;
/// <summary>
/// Class to display feedback during a course

4
Assets/MediaPipeUnity/Scripts/SignPredictor.asmdef
View File

@@ -3,12 +3,12 @@
"rootNamespace": "",
"references": [
"GUID:6055be8ebefd69e48b49212b09b47b2f",
"GUID:5c2b5ba89f9e74e418232e154bc5cc7a",
"GUID:04c4d86a70aa56c55a78c61f1ab1a56d",
"GUID:edc93f477bb73a743a97d6882ed330b3",
"GUID:58e104b97fb3752438ada2902a36dcbf",
"GUID:7f2d0ee6dd21e1d4eb25b71b7a749d25",
"GUID:f55a02e98b01bc849b30d9650ccd8f15"
"GUID:f55a02e98b01bc849b30d9650ccd8f15",
"GUID:d23f64cfd3b314bb4a18a8284c99bf5e"
],
"includePlatforms": [],
"excludePlatforms": [],

764
Assets/MediaPipeUnity/Scripts/SignPredictor.cs
View File

@@ -1,334 +1,362 @@
// Copyright (c) 2021 homuler
//
// Use of this source code is governed by an MIT-style
// license that can be found in the LICENSE file or at
// https://opensource.org/licenses/MIT.
// ATTENTION!: This code is for a tutorial.
using Mediapipe;
using Mediapipe.Unity;
using NatML;
using NatML.Features;
using NatML.Internal;
using System.Collections;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using Unity.Barracuda;
using System.Threading.Tasks;
using UnityEngine;
using UnityEngine.UI;
namespace Mediapipe.Unity.Tutorial
/// <summary>
///
/// </summary>
public class SignPredictor : MonoBehaviour
{
public class SignPredictor : MonoBehaviour
/// <summary>
/// Predictor class which is used to predict the sign using an MLEdgeModel
/// </summary>
public class NatMLSignPredictor : IMLPredictor<List<float>>
{
/// <summary>
/// ModelList, used to change model using ModelIndex
/// The MLEdgeModel used for predictions
/// </summary>
public ModelList modelList;
private readonly MLEdgeModel edgeModel;
/// <summary>
/// Reference to the model info file
/// The type used to create features which are input for the model
/// </summary>
public TextAsset modelInfoFile;
private MLFeatureType featureType;
/// <summary>
/// Config file to set up the graph
/// Creation of a NatMLSignPredictor instance
/// </summary>
[SerializeField]
private TextAsset configAsset;
/// <summary>
/// Index to indicate which camera is being used
/// </summary>
private int camdex = 0;
/// <summary>
/// The screen object on which the video is displayed
/// </summary>
[SerializeField]
private RawImage screen;
/// <summary>
/// A secondary optional screen object on which the video is displayed
/// </summary>
[SerializeField]
private RawImage screen2;
/// <summary>
/// MediaPipe graph
/// </summary>
private CalculatorGraph graph;
/// <summary>
/// Resource manager for graph resources
/// </summary>
private ResourceManager resourceManager;
/// <summary>
/// Webcam texture
/// </summary>
private WebCamTexture webcamTexture;
/// <summary>
/// Input texture
/// </summary>
private Texture2D inputTexture;
/// <summary>
/// Screen pixel data
/// </summary>
private Color32[] pixelData;
/// <summary>
/// Stopwatch to give a timestamp to video frames
/// </summary>
private Stopwatch stopwatch;
/// <summary>
/// The mediapipe stream which contains the pose landmarks
/// </summary>
private OutputStream<NormalizedLandmarkListPacket, NormalizedLandmarkList> posestream;
/// <summary>
/// The mediapipe stream which contains the left hand landmarks
/// </summary>
private OutputStream<NormalizedLandmarkListPacket, NormalizedLandmarkList> leftstream;
/// <summary>
/// The mediapipe stream which contains the right hand landmarks
/// </summary>
private OutputStream<NormalizedLandmarkListPacket, NormalizedLandmarkList> rightstream;
/// <summary>
/// create precense stream
/// </summary>
public OutputStream<DetectionVectorPacket, List<Detection>> presenceStream;
/// <summary>
/// A keypointmanager which does normalization stuff, keeps track of the landmarks
/// </summary>
private KeypointManager keypointManager;
/// <summary>
/// The worker on which we schedule the signpredictor model execution
/// </summary>
private IWorker worker;
/// <summary>
/// Width of th webcam
/// </summary>
private int width;
/// <summary>
/// Height of the webcam
/// </summary>
private int height;
/// <summary>
/// The enumerator of the worker which executes the sign predictor model
/// </summary>
private IEnumerator enumerator;
/// <summary>
/// The prediction of the sign predictor model
/// </summary>
public Dictionary<string, float> learnableProbabilities;
/// <summary>
/// Bool indicating whether or not the resource manager has already been initialized
/// </summary>
private static bool resourceManagerIsInitialized = false;
/// <summary>
/// an inputTensor for the sign predictor
/// </summary>
private Tensor inputTensor;
public List<Listener> listeners = new List<Listener>();
/// <summary>
/// Google Mediapipe setup & run
/// </summary>
/// <returns>IEnumerator</returns>
/// <exception cref="System.Exception"></exception>
private IEnumerator Start()
/// <param name="edgeModel"></param>
public NatMLSignPredictor(MLEdgeModel edgeModel)
{
// Webcam setup
if (WebCamTexture.devices.Length == 0)
{
throw new System.Exception("Web Camera devices are not found");
}
// Start the webcam
WebCamDevice webCamDevice = WebCamTexture.devices[0];
webcamTexture = new WebCamTexture(webCamDevice.name);
webcamTexture.Play();
yield return new WaitUntil(() => webcamTexture.width > 16);
// Set webcam aspect ratio
width = webcamTexture.width;
height = webcamTexture.height;
float webcamAspect = (float)webcamTexture.width / (float)webcamTexture.height;
screen.rectTransform.sizeDelta = new Vector2(screen.rectTransform.sizeDelta.y * webcamAspect, (screen.rectTransform.sizeDelta.y));
screen.texture = webcamTexture;
if (screen2 != null)
{
screen2.rectTransform.sizeDelta = new Vector2(screen2.rectTransform.sizeDelta.y * webcamAspect, (screen2.rectTransform.sizeDelta.y));
}
if (modelList.GetCurrentModel() != null)
{
// TODO this method is kinda meh you should use
inputTexture = new Texture2D(width, height, TextureFormat.RGBA32, false);
pixelData = new Color32[width * height];
if (!resourceManagerIsInitialized)
{
resourceManager = new StreamingAssetsResourceManager();
yield return resourceManager.PrepareAssetAsync("pose_detection.bytes");
yield return resourceManager.PrepareAssetAsync("pose_landmark_full.bytes");
yield return resourceManager.PrepareAssetAsync("face_landmark.bytes");
yield return resourceManager.PrepareAssetAsync("hand_landmark_full.bytes");
yield return resourceManager.PrepareAssetAsync("face_detection_short_range.bytes");
yield return resourceManager.PrepareAssetAsync("hand_recrop.bytes");
yield return resourceManager.PrepareAssetAsync("handedness.txt");
resourceManagerIsInitialized = true;
}
stopwatch = new Stopwatch();
// Setting up the graph
graph = new CalculatorGraph(configAsset.text);
posestream = new OutputStream<NormalizedLandmarkListPacket, NormalizedLandmarkList>(graph, "pose_landmarks", "pose_landmarks_presence");
leftstream = new OutputStream<NormalizedLandmarkListPacket, NormalizedLandmarkList>(graph, "left_hand_landmarks", "left_hand_landmarks_presence");
rightstream = new OutputStream<NormalizedLandmarkListPacket, NormalizedLandmarkList>(graph, "right_hand_landmarks", "right_hand_landmarks_presence");
posestream.StartPolling().AssertOk();
leftstream.StartPolling().AssertOk();
rightstream.StartPolling().AssertOk();
graph.StartRun().AssertOk();
stopwatch.Start();
keypointManager = new KeypointManager(modelInfoFile);
// check if model exists at path
//var model = ModelLoader.Load(Resources.Load<NNModel>("Models/Fingerspelling/model_A-L"));
worker = modelList.GetCurrentModel().CreateWorker();
StartCoroutine(SignRecognitionCoroutine());
StartCoroutine(MediapipeCoroutine());
}
}
/// <summary>
/// Called at the start of course/Minigame, will set the model before the start of SIgnPredictor is called.
/// </summary>
/// <param name="index">The index of the model to be used</param>
public void SetModel(ModelIndex index)
{
this.modelList.SetCurrentModel(index);
this.edgeModel = edgeModel;
featureType = edgeModel.inputs[0];
}
/// <summary>
/// Coroutine which executes the mediapipe pipeline
/// Predicts the sign using the MLEdgeModel
/// </summary>
/// <param name="inputs"></param>
/// <returns></returns>
private IEnumerator MediapipeCoroutine()
public List<float> Predict(params MLFeature[] inputs)
{
while (true)
List<float> predictions = null;
IMLEdgeFeature iedgeFeature = (IMLEdgeFeature)inputs[0];
MLEdgeFeature edgeFeature = iedgeFeature.Create(featureType);
MLFeatureCollection<MLEdgeFeature> result = edgeModel.Predict(edgeFeature);
if (0 < result.Count)
{
inputTexture.SetPixels32(webcamTexture.GetPixels32(pixelData));
var imageFrame = new ImageFrame(ImageFormat.Types.Format.Srgba, width, height, width * 4, inputTexture.GetRawTextureData<byte>());
var currentTimestamp = stopwatch.ElapsedTicks / (System.TimeSpan.TicksPerMillisecond / 1000);
graph.AddPacketToInputStream("input_video", new ImageFramePacket(imageFrame, new Timestamp(currentTimestamp))).AssertOk();
//Debug.Log(Time.timeAsDouble + " Added new packet to mediapipe graph");
yield return new WaitForEndOfFrame();
NormalizedLandmarkList _poseLandmarks = null;
NormalizedLandmarkList _leftHandLandmarks = null;
NormalizedLandmarkList _rightHandLandmarks = null;
//Debug.Log("Extracting keypoints");
yield return new WaitUntil(() => { posestream.TryGetNext(out _poseLandmarks, false); return true; });
yield return new WaitUntil(() => { leftstream.TryGetNext(out _leftHandLandmarks, false); return true; });
yield return new WaitUntil(() => { rightstream.TryGetNext(out _rightHandLandmarks, false); return true; });
//Debug.Log(Time.timeAsDouble + " Retrieved landmarks ");
keypointManager.AddLandmarks(_poseLandmarks, _leftHandLandmarks, _rightHandLandmarks);
predictions = new MLArrayFeature<float>(result[0]).Flatten().ToArray().ToList();
predictions = predictions.ConvertAll((c) => Mathf.Exp(c));
float sum = predictions.Sum();
predictions = predictions.ConvertAll((c) => c / sum);
}
edgeFeature.Dispose();
result.Dispose();
return predictions;
}
/// <summary>
/// Coroutine which calls the sign predictor model
/// Disposing the MLEdgeModel
/// </summary>
/// <returns></returns>
private IEnumerator SignRecognitionCoroutine()
public void Dispose()
{
while (true)
edgeModel.Dispose();
}
}
public List<Listener> listeners = new List<Listener>();
/// <summary>
/// Predictor which is used to create the asyncPredictor (should not be used if asyncPredictor exists)
/// </summary>
private NatMLSignPredictor predictor;
/// <summary>
/// The asynchronous predictor which is used to predict the sign using an MLEdgemodel
/// </summary>
private MLAsyncPredictor<List<float>> asyncPredictor;
/// <summary>
/// Reference to the model used in the SignPredictor
/// </summary>
private MLEdgeModel model;
/// <summary>
/// Modellist used to change model using ModelIndex
/// </summary>
public ModelList modelList;
/// <summary>
/// Chosen model data based on the operating system
/// </summary>
private MLModelData modelData;
/// <summary>
/// Reference to the model info file
/// </summary>
public TextAsset modelInfoFile;
/// <summary>
/// Config file to set up the graph
/// </summary>
[SerializeField]
private TextAsset configAsset;
/// <summary>
/// Index to indicate which camera is being used
/// </summary>
private int camdex = 0;
/// <summary>
/// The screen object on which the video is displayed
/// </summary>
[SerializeField]
private RawImage screen;
/// <summary>
/// A secondary optional screen object on which the video is displayed
/// </summary>
[SerializeField]
private RawImage screen2;
/// <summary>
/// MediaPipe graph
/// </summary>
private CalculatorGraph graph;
/// <summary>
/// Resource manager for graph resources
/// </summary>
private ResourceManager resourceManager;
/// <summary>
/// Webcam texture
/// </summary>
private WebCamTexture webcamTexture;
/// <summary>
/// Input texture
/// </summary>
private Texture2D inputTexture;
/// <summary>
/// Screen pixel data
/// </summary>
private Color32[] pixelData;
/// <summary>
/// Stopwatch to give a timestamp to video frames
/// </summary>
private Stopwatch stopwatch;
/// <summary>
/// The mediapipe stream which contains the pose landmarks
/// </summary>
private OutputStream<NormalizedLandmarkListPacket, NormalizedLandmarkList> posestream;
/// <summary>
/// The mediapipe stream which contains the left hand landmarks
/// </summary>
private OutputStream<NormalizedLandmarkListPacket, NormalizedLandmarkList> leftstream;
/// <summary>
/// The mediapipe stream which contains the right hand landmarks
/// </summary>
private OutputStream<NormalizedLandmarkListPacket, NormalizedLandmarkList> rightstream;
/// <summary>
/// create precense stream
/// </summary>
public OutputStream<DetectionVectorPacket, List<Detection>> presenceStream;
/// <summary>
/// A keypointmanager which does normalization stuff, keeps track of the landmarks
/// </summary>
private KeypointManager keypointManager;
/// <summary>
/// Width of th webcam
/// </summary>
private int width;
/// <summary>
/// Height of the webcam
/// </summary>
private int height;
/// <summary>
/// The prediction of the sign predictor model
/// </summary>
public Dictionary<string, float> learnableProbabilities;
/// <summary>
/// Bool indicating whether or not the resource manager has already been initialized
/// </summary>
private static bool resourceManagerIsInitialized = false;
/// <summary>
/// Google Mediapipe setup & run
/// </summary>
/// <returns>IEnumerator</returns>
/// <exception cref="System.Exception"></exception>
private IEnumerator Start()
{
// Webcam setup
if (WebCamTexture.devices.Length == 0)
{
throw new System.Exception("Web Camera devices are not found");
}
// Start the webcam
WebCamDevice webCamDevice = WebCamTexture.devices[0];
webcamTexture = new WebCamTexture(webCamDevice.name);
webcamTexture.Play();
yield return new WaitUntil(() => webcamTexture.width > 16);
// Set webcam aspect ratio
width = webcamTexture.width;
height = webcamTexture.height;
float webcamAspect = (float)webcamTexture.width / (float)webcamTexture.height;
screen.rectTransform.sizeDelta = new Vector2(screen.rectTransform.sizeDelta.y * webcamAspect, (screen.rectTransform.sizeDelta.y));
screen.texture = webcamTexture;
if (screen2 != null)
{
screen2.rectTransform.sizeDelta = new Vector2(screen2.rectTransform.sizeDelta.y * webcamAspect, (screen2.rectTransform.sizeDelta.y));
}
// TODO this method is kinda meh you should use
inputTexture = new Texture2D(width, height, TextureFormat.RGBA32, false);
pixelData = new Color32[width * height];
if (!resourceManagerIsInitialized)
{
resourceManager = new StreamingAssetsResourceManager();
yield return resourceManager.PrepareAssetAsync("pose_detection.bytes");
yield return resourceManager.PrepareAssetAsync("pose_landmark_full.bytes");
yield return resourceManager.PrepareAssetAsync("face_landmark.bytes");
yield return resourceManager.PrepareAssetAsync("hand_landmark_full.bytes");
yield return resourceManager.PrepareAssetAsync("face_detection_short_range.bytes");
yield return resourceManager.PrepareAssetAsync("hand_recrop.bytes");
yield return resourceManager.PrepareAssetAsync("handedness.txt");
resourceManagerIsInitialized = true;
}
stopwatch = new Stopwatch();
// Setting up the graph
graph = new CalculatorGraph(configAsset.text);
posestream = new OutputStream<NormalizedLandmarkListPacket, NormalizedLandmarkList>(graph, "pose_landmarks", "pose_landmarks_presence");
leftstream = new OutputStream<NormalizedLandmarkListPacket, NormalizedLandmarkList>(graph, "left_hand_landmarks", "left_hand_landmarks_presence");
rightstream = new OutputStream<NormalizedLandmarkListPacket, NormalizedLandmarkList>(graph, "right_hand_landmarks", "right_hand_landmarks_presence");
posestream.StartPolling().AssertOk();
leftstream.StartPolling().AssertOk();
rightstream.StartPolling().AssertOk();
graph.StartRun().AssertOk();
stopwatch.Start();
// Creating a KeypointManager
keypointManager = new KeypointManager(modelInfoFile);
// Check if a model is ready to load
yield return new WaitUntil(() => modelList.HasValidModel());
// Create Model
Task<MLEdgeModel> t = Task.Run(() => MLEdgeModel.Create(modelList.GetCurrentModel()));
yield return new WaitUntil(() => t.IsCompleted);
model = t.Result;
predictor = new NatMLSignPredictor(model);
asyncPredictor = predictor.ToAsync();
// Start the Coroutine
StartCoroutine(SignRecognitionCoroutine());
StartCoroutine(MediapipeCoroutine());
}
/// <summary>
/// Coroutine which executes the mediapipe pipeline
/// </summary>
/// <returns></returns>
private IEnumerator MediapipeCoroutine()
{
while (true)
{
inputTexture.SetPixels32(webcamTexture.GetPixels32(pixelData));
var imageFrame = new ImageFrame(ImageFormat.Types.Format.Srgba, width, height, width * 4, inputTexture.GetRawTextureData<byte>());
var currentTimestamp = stopwatch.ElapsedTicks / (System.TimeSpan.TicksPerMillisecond / 1000);
graph.AddPacketToInputStream("input_video", new ImageFramePacket(imageFrame, new Timestamp(currentTimestamp))).AssertOk();
yield return new WaitForEndOfFrame();
NormalizedLandmarkList _poseLandmarks = null;
NormalizedLandmarkList _leftHandLandmarks = null;
NormalizedLandmarkList _rightHandLandmarks = null;
yield return new WaitUntil(() => { posestream.TryGetNext(out _poseLandmarks); return true; });
yield return new WaitUntil(() => { leftstream.TryGetNext(out _leftHandLandmarks); return true; });
yield return new WaitUntil(() => { rightstream.TryGetNext(out _rightHandLandmarks); return true; });
keypointManager.AddLandmarks(_poseLandmarks, _leftHandLandmarks, _rightHandLandmarks);
}
}
/// <summary>
/// Coroutine which calls the sign predictor model
/// </summary>
/// <returns></returns>
private IEnumerator SignRecognitionCoroutine()
{
while (true)
{
List<List<float>> inputData = keypointManager.GetKeypoints();
if (inputData != null && asyncPredictor.readyForPrediction)
{
List<List<float>> input = keypointManager.GetKeypoints();
if (input != null)
// Getting the size of the input data
int framecount = inputData.Count;
int keypointsPerFrame = inputData[0].Count;
// Creating ArrayFeature
int[] shape = { framecount, keypointsPerFrame };
float[] input = new float[framecount * keypointsPerFrame];
int i = 0;
inputData.ForEach((e) => e.ForEach((f) => input[i++] = f));
MLArrayFeature<float> feature = new MLArrayFeature<float>(input, shape);
// Predicting
Task<List<float>> task = Task.Run(async () => await asyncPredictor.Predict(feature));
yield return new WaitUntil(() => task.IsCompleted);
List<float> result = task.Result;
if (0 < result.Count)
{
//UnityEngine.Debug.Log("input: " + input.Count);
int frameCount = input.Count;
int keypoints_per_frame = input[0].Count;
// Create a tensor with the input
inputTensor = new Tensor(frameCount, keypoints_per_frame);
// Fill the tensor with the input
for (int i = 0; i < frameCount; i++)
{
for (int j = 0; j < keypoints_per_frame; j++)
{
inputTensor[i, j] = input[i][j];
}
}
int stepsPerFrame = 190;
enumerator = worker.StartManualSchedule(inputTensor);
int step = 0;
while (enumerator.MoveNext())
{
if (++step % stepsPerFrame == 0)
{
//Debug.Log(Time.timeAsDouble + " : " + step);
yield return null;
}
}
var output = worker.PeekOutput();
inputTensor.Dispose();
// Get the output as an array
float[] outputArray = output.ToReadOnlyArray();
//Debug.Log($"out = [{outputArray.Aggregate(" ", (t, f) => $"{t}{f} ")}]");
// Calculate the softmax of the output
float max = outputArray.Max();
float[] softmaxedOutput = outputArray.Select(x => Mathf.Exp(x - max)).ToArray();
float sum = softmaxedOutput.Sum();
float[] softmaxedOutput2 = softmaxedOutput.Select(x => x / sum).ToArray();
// Get the index of the highest probability
int maxIndex = softmaxedOutput2.ToList().IndexOf(softmaxedOutput2.Max());
// Get the letter from the index
char letter = (char)(maxIndex + 65);
float accuracy = (Mathf.RoundToInt(softmaxedOutput2[maxIndex] * 100));
// Set the letterProbabilities, currently used by Courses
learnableProbabilities = new Dictionary<string, float>();
for (int i = 0; i < softmaxedOutput2.Length; i++)
// Temporary fix
List<string> signs = new List<string>()
{
learnableProbabilities.Add(((char)(i + 65)).ToString(), softmaxedOutput2[i]);
"A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M",
"N", "O", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z"
};
for (int j = 0; j < result.Count; j++)
{
learnableProbabilities.Add(signs[j].ToUpper(), result[j]);
}
//Debug.Log($"prob = [{learnableProbabilities.Aggregate(" ", (t, kv) => $"{t}{kv.Key}:{kv.Value} ")}]");
foreach(Listener listener in listeners)
foreach (Listener listener in listeners)
{
yield return listener.ProcessIncomingCall();
}
@@ -339,77 +367,85 @@ namespace Mediapipe.Unity.Tutorial
yield return null;
}
}
}
/// <summary>
/// Propper destruction on the Mediapipegraph
/// </summary>
private void OnDestroy()
{
if (webcamTexture != null)
{
webcamTexture.Stop();
}
if (graph != null)
{
try
{
graph.CloseInputStream("input_video").AssertOk();
graph.WaitUntilDone().AssertOk();
}
finally
{
graph.Dispose();
}
}
// inputTensor must still be disposed, if it exists
inputTensor?.Dispose();
worker?.Dispose();
}
/// <summary>
/// So long as there are cameras to use, you swap the camera you are using to another in the list.
/// </summary>
public void SwapCam()
{
if (WebCamTexture.devices.Length > 0)
{
// Stop the old camera
// If there was no camera playing before, then you dont have to reset the texture, as it wasn't assigned in the first place.
if (webcamTexture.isPlaying)
{
screen.texture = null;
webcamTexture.Stop();
webcamTexture = null;
}
// Find the new camera
camdex += 1;
camdex %= WebCamTexture.devices.Length;
// Start the new camera
WebCamDevice device = WebCamTexture.devices[camdex];
webcamTexture = new WebCamTexture(device.name);
screen.texture = webcamTexture;
webcamTexture.Play();
}
}
/// <summary>
/// Swaps the display screens
/// </summary>
public void SwapScreen()
{
if(screen2.texture == null && screen.texture != null)
{
screen2.texture = webcamTexture;
screen.texture = null;
}
else if (screen2.texture != null && screen.texture == null)
{
screen.texture = webcamTexture;
screen2.texture = null;
}
yield return null;
}
}
/// <summary>
/// Propper destruction on the Mediapipegraph
/// </summary>
private void OnDestroy()
{
if (webcamTexture != null)
{
webcamTexture.Stop();
}
if (graph != null)
{
try
{
graph.CloseInputStream("input_video").AssertOk();
graph.WaitUntilDone().AssertOk();
}
finally
{
graph.Dispose();
}
}
if (asyncPredictor != null)
{
asyncPredictor.Dispose();
}
}
/// <summary>
/// So long as there are cameras to use, you swap the camera you are using to another in the list.
/// </summary>
public void SwapCam()
{
if (WebCamTexture.devices.Length > 0)
{
// Stop the old camera
// If there was no camera playing before, then you dont have to reset the texture, as it wasn't assigned in the first place.
if (webcamTexture.isPlaying)
{
screen.texture = null;
webcamTexture.Stop();
webcamTexture = null;
}
// Find the new camera
camdex += 1;
camdex %= WebCamTexture.devices.Length;
// Start the new camera
WebCamDevice device = WebCamTexture.devices[camdex];
webcamTexture = new WebCamTexture(device.name);
screen.texture = webcamTexture;
webcamTexture.Play();
}
}
/// <summary>
/// Swaps the display screens
/// </summary>
public void SwapScreen()
{
if (screen2.texture == null && screen.texture != null)
{
screen2.texture = webcamTexture;
screen.texture = null;
}
else if (screen2.texture != null && screen.texture == null)
{
screen.texture = webcamTexture;
screen2.texture = null;
}
}
public void ChangeModel(ModelIndex index)
{
this.modelList.SetCurrentModel(index);
}
}

2
Assets/SpellingBee/Scripts/SpellingBeeController.cs
View File

@@ -179,7 +179,7 @@ public partial class SpellingBeeController : AbstractFeedback
{
StartController();
signPredictor.SetModel(currentTheme.modelIndex);
signPredictor.ChangeModel(ModelIndex.FINGERSPELLING);
AddSelfAsListener();
}
/// <summary>