WeSign/unity-application
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 6 Wiki Activity

Resolve WES-97 "Integrate signpredictor in spellingbee"

Browse Source
This commit is contained in:
Dries Van Schuylenbergh
2023-03-19 17:37:50 +00:00
committed by Lukas Van Rossem
parent f827c29d3a
commit 3abc24a39c
72 changed files with 3169 additions and 1886 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
Assets/MediaPipeUnity/Common/Scripts/KeypointManager.cs
View File

@@ -1,7 +1,7 @@
using System.Collections;
using Mediapipe;
using System.Collections.Generic;
using UnityEngine;
using System.Linq;
using UnityEngine;
public class ModelInfo
{
@@ -12,17 +12,16 @@ public class ModelInfo
public class KeypointManager
{
private ModelInfo model_info;
private List<List<float>> keypoints_buffer;
private ModelInfo modelInfo;
private List<List<float>> keypointsBuffer;
public KeypointManager()
public KeypointManager(TextAsset modelInfoFile)
{
TextAsset model_info_json = Resources.Load<TextAsset>("Models/FingerSpelling/landmarks");
this.model_info = JsonUtility.FromJson<ModelInfo>(model_info_json.text);
this.keypoints_buffer = new List<List<float>>();
modelInfo = JsonUtility.FromJson<ModelInfo>(modelInfoFile.text);
keypointsBuffer = new List<List<float>>();
}
private (List<float>, List<float>) normalizeHand(List<float> hand_x, List<float> hand_y)
private (List<float>, List<float>) NormalizeHand(List<float> hand_x, List<float> hand_y)
{
float min_x = hand_x.Min();
@@ -54,7 +53,7 @@ public class KeypointManager
return (normalized_x, normalized_y);
}
public void addLandmarks(Mediapipe.NormalizedLandmarkList poseLandmarks, Mediapipe.NormalizedLandmarkList leftHandLandmarks, Mediapipe.NormalizedLandmarkList rightHandLandmarks)
public void AddLandmarks(NormalizedLandmarkList poseLandmarks, NormalizedLandmarkList leftHandLandmarks, NormalizedLandmarkList rightHandLandmarks)
{
List<float> pose_x = new List<float>();
List<float> pose_y = new List<float>();
@@ -65,7 +64,7 @@ public class KeypointManager
if (poseLandmarks != null)
{
foreach (var landmark_index in model_info.pose_landmarks)
foreach (int landmark_index in modelInfo.pose_landmarks)
{
pose_x.Add(poseLandmarks.Landmark[landmark_index].X);
pose_y.Add(poseLandmarks.Landmark[landmark_index].Y);
@@ -73,7 +72,7 @@ public class KeypointManager
}
else
{
foreach (var landmark_index in model_info.pose_landmarks)
foreach (int _ in modelInfo.pose_landmarks)
{
pose_x.Add(0);
pose_y.Add(0);
@@ -82,7 +81,7 @@ public class KeypointManager
foreach (var landmark_index in model_info.hand_landmarks)
foreach (int landmark_index in modelInfo.hand_landmarks)
{
if (leftHandLandmarks == null)
{
@@ -107,13 +106,15 @@ public class KeypointManager
}
// TODO: Add normalization
(left_hand_x, left_hand_y) = normalizeHand(left_hand_x, left_hand_y);
(right_hand_x, right_hand_y) = normalizeHand(right_hand_x, right_hand_y);
//Debug.Log($"pose_landMarks = [{modelInfo.pose_landmarks.Aggregate(" ", (t, f) => $"{t} {f}")}]");
//Debug.Log($"hand_landmarks = [{modelInfo.hand_landmarks.Aggregate(" ", (t, f) => $"{t} {f}")}]");
(left_hand_x, left_hand_y) = NormalizeHand(left_hand_x, left_hand_y);
(right_hand_x, right_hand_y) = NormalizeHand(right_hand_x, right_hand_y);
if (keypoints_buffer.Count >= 10)
if (keypointsBuffer.Count >= 10)
{
keypoints_buffer.RemoveAt(0);
keypointsBuffer.RemoveAt(0);
}
List<float> keypoints = new List<float>();
@@ -133,15 +134,15 @@ public class KeypointManager
keypoints.Add(right_hand_y[i]);
}
keypoints_buffer.Add(keypoints);
keypointsBuffer.Add(keypoints);
}
public List<List<float>> getAllKeypoints()
public List<List<float>> GetKeypoints()
{
if (keypoints_buffer.Count < 10)
if (keypointsBuffer.Count < 10)
{
return null;
}
return keypoints_buffer;
return keypointsBuffer;
}
}

8
Assets/MediaPipeUnity/Prefabs.meta Normal file
View File

@@ -0,0 +1,8 @@
fileFormatVersion: 2
guid: f985e6b2f4506ff45a14a0d651610b0d
folderAsset: yes
DefaultImporter:
externalObjects: {}
userData:
assetBundleName:
assetBundleVariant:

471
Assets/MediaPipeUnity/Prefabs/Feedback.prefab Normal file
View File

@@ -0,0 +1,471 @@
%YAML 1.1
%TAG !u! tag:unity3d.com,2011:
--- !u!1 &4318122119930585319
GameObject:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
serializedVersion: 6
m_Component:
- component: {fileID: 4318122119930585316}
- component: {fileID: 4318122119930585317}
m_Layer: 5
m_Name: Feedback
m_TagString: Untagged
m_Icon: {fileID: 0}
m_NavMeshLayer: 0
m_StaticEditorFlags: 0
m_IsActive: 1
--- !u!224 &4318122119930585316
RectTransform:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 4318122119930585319}
m_LocalRotation: {x: 0, y: 0, z: 0, w: 1}
m_LocalPosition: {x: 0, y: 0, z: 0}
m_LocalScale: {x: 1, y: 1, z: 1}
m_ConstrainProportionsScale: 0
m_Children:
- {fileID: 4318122120222767926}
- {fileID: 4318122119968934245}
m_Father: {fileID: 0}
m_RootOrder: 0
m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
m_AnchorMin: {x: 0.5, y: 0}
m_AnchorMax: {x: 0.5, y: 0}
m_AnchoredPosition: {x: 0, y: 200}
m_SizeDelta: {x: 500, y: 150}
m_Pivot: {x: 0.5, y: 0}
--- !u!114 &4318122119930585317
MonoBehaviour:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 4318122119930585319}
m_Enabled: 1
m_EditorHideFlags: 0
m_Script: {fileID: 11500000, guid: 44e682a32ee15cc489bf50f3a06f717b, type: 3}
m_Name:
m_EditorClassIdentifier:
feedbackText: {fileID: 4318122120222767928}
feedbackProgress: {fileID: 4318122119968934242}
feedbackProgressImage: {fileID: 4318122120334233319}
signPredictor: {fileID: 0}
--- !u!1 &4318122119968934244
GameObject:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
serializedVersion: 6
m_Component:
- component: {fileID: 4318122119968934245}
- component: {fileID: 4318122119968934242}
m_Layer: 5
m_Name: Progress
m_TagString: Untagged
m_Icon: {fileID: 0}
m_NavMeshLayer: 0
m_StaticEditorFlags: 0
m_IsActive: 1
--- !u!224 &4318122119968934245
RectTransform:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 4318122119968934244}
m_LocalRotation: {x: 0, y: 0, z: 0, w: 1}
m_LocalPosition: {x: 0, y: 0, z: 0}
m_LocalScale: {x: 1, y: 1, z: 1}
m_ConstrainProportionsScale: 0
m_Children:
- {fileID: 4318122120783817558}
- {fileID: 4318122121620266793}
m_Father: {fileID: 4318122119930585316}
m_RootOrder: 1
m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
m_AnchorMin: {x: 0, y: 0}
m_AnchorMax: {x: 1, y: 0}
m_AnchoredPosition: {x: 0, y: 0}
m_SizeDelta: {x: 0, y: 50}
m_Pivot: {x: 0.5, y: 0}
--- !u!114 &4318122119968934242
MonoBehaviour:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 4318122119968934244}
m_Enabled: 1
m_EditorHideFlags: 0
m_Script: {fileID: 11500000, guid: 67db9e8f0e2ae9c40bc1e2b64352a6b4, type: 3}
m_Name:
m_EditorClassIdentifier:
m_Navigation:
m_Mode: 3
m_WrapAround: 0
m_SelectOnUp: {fileID: 0}
m_SelectOnDown: {fileID: 0}
m_SelectOnLeft: {fileID: 0}
m_SelectOnRight: {fileID: 0}
m_Transition: 1
m_Colors:
m_NormalColor: {r: 1, g: 1, b: 1, a: 1}
m_HighlightedColor: {r: 0.9607843, g: 0.9607843, b: 0.9607843, a: 1}
m_PressedColor: {r: 0.78431374, g: 0.78431374, b: 0.78431374, a: 1}
m_SelectedColor: {r: 0.9607843, g: 0.9607843, b: 0.9607843, a: 1}
m_DisabledColor: {r: 0.78431374, g: 0.78431374, b: 0.78431374, a: 0.5019608}
m_ColorMultiplier: 1
m_FadeDuration: 0.1
m_SpriteState:
m_HighlightedSprite: {fileID: 0}
m_PressedSprite: {fileID: 0}
m_SelectedSprite: {fileID: 0}
m_DisabledSprite: {fileID: 0}
m_AnimationTriggers:
m_NormalTrigger: Normal
m_HighlightedTrigger: Highlighted
m_PressedTrigger: Pressed
m_SelectedTrigger: Selected
m_DisabledTrigger: Disabled
m_Interactable: 1
m_TargetGraphic: {fileID: 0}
m_FillRect: {fileID: 4318122120334233317}
m_HandleRect: {fileID: 0}
m_Direction: 0
m_MinValue: 0
m_MaxValue: 1
m_WholeNumbers: 0
m_Value: 0
m_OnValueChanged:
m_PersistentCalls:
m_Calls: []
--- !u!1 &4318122120222767931
GameObject:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
serializedVersion: 6
m_Component:
- component: {fileID: 4318122120222767926}
- component: {fileID: 4318122120222767929}
- component: {fileID: 4318122120222767928}
m_Layer: 5
m_Name: Text
m_TagString: Untagged
m_Icon: {fileID: 0}
m_NavMeshLayer: 0
m_StaticEditorFlags: 0
m_IsActive: 1
--- !u!224 &4318122120222767926
RectTransform:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 4318122120222767931}
m_LocalRotation: {x: -0, y: -0, z: -0, w: 1}
m_LocalPosition: {x: 0, y: 0, z: 0}
m_LocalScale: {x: 1, y: 1, z: 1}
m_ConstrainProportionsScale: 0
m_Children: []
m_Father: {fileID: 4318122119930585316}
m_RootOrder: 0
m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
m_AnchorMin: {x: 0.5, y: 1}
m_AnchorMax: {x: 0.5, y: 1}
m_AnchoredPosition: {x: 0, y: 0}
m_SizeDelta: {x: 500, y: 100}
m_Pivot: {x: 0.5, y: 1}
--- !u!222 &4318122120222767929
CanvasRenderer:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 4318122120222767931}
m_CullTransparentMesh: 1
--- !u!114 &4318122120222767928
MonoBehaviour:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 4318122120222767931}
m_Enabled: 1
m_EditorHideFlags: 0
m_Script: {fileID: 11500000, guid: f4688fdb7df04437aeb418b961361dc5, type: 3}
m_Name:
m_EditorClassIdentifier:
m_Material: {fileID: 0}
m_Color: {r: 1, g: 1, b: 1, a: 1}
m_RaycastTarget: 1
m_RaycastPadding: {x: 0, y: 0, z: 0, w: 0}
m_Maskable: 1
m_OnCullStateChanged:
m_PersistentCalls:
m_Calls: []
m_text: Detecteren ...
m_isRightToLeft: 0
m_fontAsset: {fileID: 11400000, guid: 8f586378b4e144a9851e7b34d9b748ee, type: 2}
m_sharedMaterial: {fileID: 2180264, guid: 8f586378b4e144a9851e7b34d9b748ee, type: 2}
m_fontSharedMaterials: []
m_fontMaterial: {fileID: 0}
m_fontMaterials: []
m_fontColor32:
serializedVersion: 2
rgba: 4282188031
m_fontColor: {r: 1, g: 0, b: 0.23945475, a: 1}
m_enableVertexGradient: 0
m_colorMode: 3
m_fontColorGradient:
topLeft: {r: 1, g: 1, b: 1, a: 1}
topRight: {r: 1, g: 1, b: 1, a: 1}
bottomLeft: {r: 1, g: 1, b: 1, a: 1}
bottomRight: {r: 1, g: 1, b: 1, a: 1}
m_fontColorGradientPreset: {fileID: 0}
m_spriteAsset: {fileID: 0}
m_tintAllSprites: 0
m_StyleSheet: {fileID: 0}
m_TextStyleHashCode: -1183493901
m_overrideHtmlColors: 0
m_faceColor:
serializedVersion: 2
rgba: 4294967295
m_fontSize: 48
m_fontSizeBase: 48
m_fontWeight: 400
m_enableAutoSizing: 0
m_fontSizeMin: 18
m_fontSizeMax: 72
m_fontStyle: 1
m_HorizontalAlignment: 2
m_VerticalAlignment: 512
m_textAlignment: 65535
m_characterSpacing: 0
m_wordSpacing: 0
m_lineSpacing: 0
m_lineSpacingMax: 0
m_paragraphSpacing: 0
m_charWidthMaxAdj: 0
m_enableWordWrapping: 1
m_wordWrappingRatios: 0.4
m_overflowMode: 0
m_linkedTextComponent: {fileID: 0}
parentLinkedComponent: {fileID: 0}
m_enableKerning: 1
m_enableExtraPadding: 0
checkPaddingRequired: 0
m_isRichText: 1
m_parseCtrlCharacters: 1
m_isOrthographic: 1
m_isCullingEnabled: 0
m_horizontalMapping: 0
m_verticalMapping: 0
m_uvLineOffset: 0
m_geometrySortingOrder: 0
m_IsTextObjectScaleStatic: 0
m_VertexBufferAutoSizeReduction: 0
m_useMaxVisibleDescender: 1
m_pageToDisplay: 1
m_margin: {x: 0, y: 0, z: 0, w: 0}
m_isUsingLegacyAnimationComponent: 0
m_isVolumetricText: 0
m_hasFontAssetChanged: 0
m_baseMaterial: {fileID: 0}
m_maskOffset: {x: 0, y: 0, z: 0, w: 0}
--- !u!1 &4318122120334233318
GameObject:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
serializedVersion: 6
m_Component:
- component: {fileID: 4318122120334233317}
- component: {fileID: 4318122120334233316}
- component: {fileID: 4318122120334233319}
m_Layer: 5
m_Name: Fill
m_TagString: Untagged
m_Icon: {fileID: 0}
m_NavMeshLayer: 0
m_StaticEditorFlags: 0
m_IsActive: 1
--- !u!224 &4318122120334233317
RectTransform:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 4318122120334233318}
m_LocalRotation: {x: -0, y: -0, z: -0, w: 1}
m_LocalPosition: {x: 0, y: 0, z: 0}
m_LocalScale: {x: 1, y: 1, z: 1}
m_ConstrainProportionsScale: 0
m_Children: []
m_Father: {fileID: 4318122121620266793}
m_RootOrder: 0
m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
m_AnchorMin: {x: 0, y: 0}
m_AnchorMax: {x: 0, y: 0}
m_AnchoredPosition: {x: 0, y: 0}
m_SizeDelta: {x: 10, y: 0}
m_Pivot: {x: 0.5, y: 0.5}
--- !u!222 &4318122120334233316
CanvasRenderer:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 4318122120334233318}
m_CullTransparentMesh: 1
--- !u!114 &4318122120334233319
MonoBehaviour:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 4318122120334233318}
m_Enabled: 1
m_EditorHideFlags: 0
m_Script: {fileID: 11500000, guid: fe87c0e1cc204ed48ad3b37840f39efc, type: 3}
m_Name:
m_EditorClassIdentifier:
m_Material: {fileID: 0}
m_Color: {r: 1, g: 0, b: 0, a: 1}
m_RaycastTarget: 1
m_RaycastPadding: {x: 0, y: 0, z: 0, w: 0}
m_Maskable: 1
m_OnCullStateChanged:
m_PersistentCalls:
m_Calls: []
m_Sprite: {fileID: 10905, guid: 0000000000000000f000000000000000, type: 0}
m_Type: 1
m_PreserveAspect: 0
m_FillCenter: 1
m_FillMethod: 4
m_FillAmount: 1
m_FillClockwise: 1
m_FillOrigin: 0
m_UseSpriteMesh: 0
m_PixelsPerUnitMultiplier: 1
--- !u!1 &4318122120783817561
GameObject:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
serializedVersion: 6
m_Component:
- component: {fileID: 4318122120783817558}
- component: {fileID: 4318122120783817556}
- component: {fileID: 4318122120783817559}
m_Layer: 5
m_Name: Background
m_TagString: Untagged
m_Icon: {fileID: 0}
m_NavMeshLayer: 0
m_StaticEditorFlags: 0
m_IsActive: 1
--- !u!224 &4318122120783817558
RectTransform:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 4318122120783817561}
m_LocalRotation: {x: -0, y: -0, z: -0, w: 1}
m_LocalPosition: {x: 0, y: 0, z: 0}
m_LocalScale: {x: 1, y: 1, z: 1}
m_ConstrainProportionsScale: 0
m_Children: []
m_Father: {fileID: 4318122119968934245}
m_RootOrder: 0
m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
m_AnchorMin: {x: 0, y: 0}
m_AnchorMax: {x: 1, y: 1}
m_AnchoredPosition: {x: 0, y: 0}
m_SizeDelta: {x: 0, y: 0}
m_Pivot: {x: 0.5, y: 0.5}
--- !u!222 &4318122120783817556
CanvasRenderer:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 4318122120783817561}
m_CullTransparentMesh: 1
--- !u!114 &4318122120783817559
MonoBehaviour:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 4318122120783817561}
m_Enabled: 1
m_EditorHideFlags: 0
m_Script: {fileID: 11500000, guid: fe87c0e1cc204ed48ad3b37840f39efc, type: 3}
m_Name:
m_EditorClassIdentifier:
m_Material: {fileID: 0}
m_Color: {r: 1, g: 1, b: 1, a: 1}
m_RaycastTarget: 1
m_RaycastPadding: {x: 0, y: 0, z: 0, w: 0}
m_Maskable: 1
m_OnCullStateChanged:
m_PersistentCalls:
m_Calls: []
m_Sprite: {fileID: 10907, guid: 0000000000000000f000000000000000, type: 0}
m_Type: 1
m_PreserveAspect: 0
m_FillCenter: 1
m_FillMethod: 4
m_FillAmount: 1
m_FillClockwise: 1
m_FillOrigin: 0
m_UseSpriteMesh: 0
m_PixelsPerUnitMultiplier: 1
--- !u!1 &4318122121620266792
GameObject:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
serializedVersion: 6
m_Component:
- component: {fileID: 4318122121620266793}
m_Layer: 5
m_Name: Fill Area
m_TagString: Untagged
m_Icon: {fileID: 0}
m_NavMeshLayer: 0
m_StaticEditorFlags: 0
m_IsActive: 1
--- !u!224 &4318122121620266793
RectTransform:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 4318122121620266792}
m_LocalRotation: {x: -0, y: -0, z: -0, w: 1}
m_LocalPosition: {x: 0, y: 0, z: 0}
m_LocalScale: {x: 1, y: 1, z: 1}
m_ConstrainProportionsScale: 0
m_Children:
- {fileID: 4318122120334233317}
m_Father: {fileID: 4318122119968934245}
m_RootOrder: 1
m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
m_AnchorMin: {x: 0, y: 0}
m_AnchorMax: {x: 1, y: 1}
m_AnchoredPosition: {x: 0, y: 0}
m_SizeDelta: {x: 0, y: 0}
m_Pivot: {x: 0.5, y: 0.5}

7
Assets/MediaPipeUnity/Prefabs/Feedback.prefab.meta Normal file
View File

@@ -0,0 +1,7 @@
fileFormatVersion: 2
guid: 7c71c65ecb5fe0449a8b0d178987f016
PrefabImporter:
externalObjects: {}
userData:
assetBundleName:
assetBundleVariant:

180
Assets/MediaPipeUnity/Scripts/Feedback.cs Normal file
View File

@@ -0,0 +1,180 @@
using DigitalRuby.Tween;
using Mediapipe.Unity.Tutorial;
using System;
using System.Collections;
using TMPro;
using UnityEngine;
using UnityEngine.Events;
using UnityEngine.UI;
/// <summary>
/// Class to display feedback during a course
/// </summary>
public class Feedback : MonoBehaviour
{
/// <summary>
/// Reference to the feedback field
/// </summary>
public TMP_Text feedbackText;
/// <summary>
/// Reference to the progress bar
/// </summary>
public Slider feedbackProgress;
/// <summary>
/// Reference to the progress bar image, so we can add fancy colors
/// </summary>
public Image feedbackProgressImage;
/// <summary>
/// Reference to the sign predictor
/// </summary>
public SignPredictor signPredictor;
/// <summary>
/// Callback for getting the correct sign
/// </summary>
public Func<string> getSignCallback;
/// <summary>
/// Callback to initiate the next sign
/// </summary>
public UnityAction<string> predictSignCallback;
/// <summary>
/// Timer to keep track of how long a incorrect sign is performed
/// </summary>
private DateTime timer;
/// <summary>
/// Current predicted sign
/// </summary>
private string predictedSign = null;
/// <summary>
/// Previous incorrect sign, so we can keep track whether the user is wrong or the user is still changing signs
/// </summary>
private string previousIncorrectSign = null;
/// <summary>
/// Start is called before the first frame update
/// </summary>
void Start()
{
// Start the coroutine to update the scale every 200 milliseconds
StartCoroutine(UpdateFeedback());
}
/// <summary>
/// UpdateScale updates the progress bar every 200ms, updated the feedback text, and progress bar color
/// If a high enough accuracy is detected, it will go to the next sign
/// </summary>
/// <returns></returns>
IEnumerator UpdateFeedback()
{
while (true)
{
if (getSignCallback != null && predictSignCallback != null)
{
// Get current sign
string currentSign = getSignCallback();
// Get the predicted sign
if (signPredictor != null && signPredictor.learnableProbabilities != null &&
currentSign != null && signPredictor.learnableProbabilities.ContainsKey(currentSign))
{
float accuracy = signPredictor.learnableProbabilities[currentSign];
if (accuracy > 0.98)
{
// TODO: fix emojis
feedbackText.text = "✨ Perfect ✨";
Color col = new Color(0xff / 255.0f, 0xcc / 255.0f, 0x00 / 255.0f);
feedbackText.color = col;
feedbackProgressImage.color = col;
}
else if (accuracy > 0.95)
{
feedbackText.text = "Super!";
Color col = new Color(0x00 / 255.0f, 0xff / 255.0f, 0xcc / 255.0f);
feedbackText.color = col;
feedbackProgressImage.color = col;
}
else if (accuracy > 0.90)
{
feedbackText.text = "Goed";
feedbackText.color = Color.green;
feedbackProgressImage.color = Color.green;
}
else if (accuracy > 0.80)
{
feedbackText.text = "Bijna...";
Color col = new Color(0xff / 255.0f, 0x66 / 255.0f, 0x00 / 255.0f);
feedbackText.color = col;
feedbackProgressImage.color = col;
}
else
{
feedbackText.text = "Detecteren...";
feedbackText.color = Color.red;
feedbackProgressImage.color = Color.red;
}
float oldValue = feedbackProgress.value;
// use an exponential scale
float newValue = Mathf.Exp(4 * (accuracy - 1.0f));
feedbackProgress.gameObject.Tween("FeedbackUpdate", oldValue, newValue, 0.2f, TweenScaleFunctions.CubicEaseInOut, (t) =>
{
if (feedbackProgress != null)
{
feedbackProgress.value = t.CurrentValue;
}
});
// Check whether (in)correct sign has high accuracy
foreach (var kv in signPredictor.learnableProbabilities)
{
if (kv.Value > 0.90)
{
predictedSign = kv.Key;
// Correct sign
if (predictedSign == currentSign)
{
yield return new WaitForSeconds(1.0f);
predictSignCallback(predictedSign);
timer = DateTime.Now;
predictedSign = null;
previousIncorrectSign = null;
}
// Incorrect sign
else
{
if (previousIncorrectSign != predictedSign)
{
timer = DateTime.Now;
previousIncorrectSign = predictedSign;
}
else if (DateTime.Now - timer > TimeSpan.FromSeconds(2.0f))
{
predictSignCallback(predictedSign);
timer = DateTime.Now;
predictedSign = null;
previousIncorrectSign = null;
}
}
break;
}
}
}
else
{
feedbackProgress.value = 0.0f;
}
}
// Wait for 200 milliseconds before updating the scale again
yield return new WaitForSeconds(0.2f);
}
}
}

11
Assets/MediaPipeUnity/Scripts/Feedback.cs.meta Normal file
View File

@@ -0,0 +1,11 @@
fileFormatVersion: 2
guid: 44e682a32ee15cc489bf50f3a06f717b
MonoImporter:
externalObjects: {}
serializedVersion: 2
defaultReferences: []
executionOrder: 0
icon: {instanceID: 0}
userData:
assetBundleName:
assetBundleVariant:

3
Assets/MediaPipeUnity/Scripts/SignPredictor.asmdef
View File

@@ -5,7 +5,8 @@
"GUID:6055be8ebefd69e48b49212b09b47b2f",
"GUID:5c2b5ba89f9e74e418232e154bc5cc7a",
"GUID:04c4d86a70aa56c55a78c61f1ab1a56d",
"GUID:edc93f477bb73a743a97d6882ed330b3"
"GUID:edc93f477bb73a743a97d6882ed330b3",
"GUID:58e104b97fb3752438ada2902a36dcbf"
],
"includePlatforms": [],
"excludePlatforms": [],

187
Assets/MediaPipeUnity/Scripts/Wesign_extractor.cs → Assets/MediaPipeUnity/Scripts/SignPredictor.cs
View File

@@ -10,55 +10,70 @@ using System.Collections;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using TMPro;
using Unity.Barracuda;
using UnityEngine;
using UnityEngine.UI;
using Debug = UnityEngine.Debug;
namespace Mediapipe.Unity.Tutorial
{
public class Wesign_extractor : MonoBehaviour
public class SignPredictor : MonoBehaviour
{
/// <summary>
/// Reference to the model used in the SignPredictor
/// </summary>
public NNModel model;
/// <summary>
/// Reference to the model info file
/// </summary>
public TextAsset modelInfoFile;
/// <summary>
/// Config file to set up the graph
/// </summary>
[SerializeField] private TextAsset _configAsset;
[SerializeField]
private TextAsset configAsset;
/// <summary>
/// Index to indicate which camera is being used
/// </summary>
private int camdex = 0;
/// <summary>
/// The screen object on which the video is displayed
/// </summary>
[SerializeField] private RawImage _screen;
[SerializeField]
private RawImage screen;
/// <summary>
/// MediaPipe graph
/// </summary>
private CalculatorGraph _graph;
private CalculatorGraph graph;
/// <summary>
/// Resource manager for graph resources
/// </summary>
private ResourceManager _resourceManager;
private ResourceManager resourceManager;
/// <summary>
/// Webcam texture
/// </summary>
private WebCamTexture _webCamTexture;
private WebCamTexture webcamTexture;
/// <summary>
/// Input texture
/// </summary>
private Texture2D _inputTexture;
private Texture2D inputTexture;
/// <summary>
/// Screen pixel data
/// </summary>
private Color32[] _pixelData;
private Color32[] pixelData;
/// <summary>
/// Stopwatch to give a timestamp to video frames
/// </summary>
private Stopwatch _stopwatch;
private Stopwatch stopwatch;
/// <summary>
/// The mediapipe stream which contains the pose landmarks
@@ -78,12 +93,12 @@ namespace Mediapipe.Unity.Tutorial
/// <summary>
/// create precense stream
/// </summary>
public OutputStream<DetectionVectorPacket, List<Detection>> _presenceStream;
public OutputStream<DetectionVectorPacket, List<Detection>> presenceStream;
/// <summary>
/// A keypointmanager which does normalization stuff, keeps track of the landmarks
/// </summary>
private KeypointManager k;
private KeypointManager keypointManager;
/// <summary>
/// The worker on which we schedule the signpredictor model execution
@@ -93,22 +108,12 @@ namespace Mediapipe.Unity.Tutorial
/// <summary>
/// Width of th webcam
/// </summary>
private int _width;
private int width;
/// <summary>
/// Height of the webcam
/// </summary>
private int _height;
/// <summary>
/// ?The mediapipe stream which contains the tracked detections
/// </summary>
private const string _TrackedDetectionsStreamName = "tracked_detections";
/// <summary>
/// ?The mediapipe stream which contains the tracked detections
/// </summary>
private OutputStream<DetectionVectorPacket, List<Detection>> _trackedDetectionsStream;
private int height;
/// <summary>
/// The enumerator of the worker which executes the sign predictor model
@@ -118,7 +123,7 @@ namespace Mediapipe.Unity.Tutorial
/// <summary>
/// The prediction of the sign predictor model
/// </summary>
public Dictionary<char, float> letterProbabilities;
public Dictionary<string, float> learnableProbabilities;
/// <summary>
/// Bool indicating whether or not the resource manager has already been initialized
@@ -133,12 +138,10 @@ namespace Mediapipe.Unity.Tutorial
/// <summary>
/// Google Mediapipe setup & run
/// </summary>
/// <returns> IEnumerator </returns>
/// <returns>IEnumerator</returns>
/// <exception cref="System.Exception"></exception>
private IEnumerator Start()
{
Debug.Log("starting ...");
// Webcam setup
if (WebCamTexture.devices.Length == 0)
{
@@ -146,57 +149,57 @@ namespace Mediapipe.Unity.Tutorial
}
// Start the webcam
WebCamDevice webCamDevice = WebCamTexture.devices[0];
_webCamTexture = new WebCamTexture(webCamDevice.name);
webcamTexture = new WebCamTexture(webCamDevice.name);
_webCamTexture.Play();
webcamTexture.Play();
yield return new WaitUntil(() => _webCamTexture.width > 16);
yield return new WaitUntil(() => webcamTexture.width > 16);
// Set webcam aspect ratio
_width = _webCamTexture.width;
_height = _webCamTexture.height;
float webcamAspect = (float)_webCamTexture.width / (float)_webCamTexture.height;
_screen.rectTransform.sizeDelta = new Vector2(_screen.rectTransform.sizeDelta.y * webcamAspect, (_screen.rectTransform.sizeDelta.y));
_screen.texture = _webCamTexture;
width = webcamTexture.width;
height = webcamTexture.height;
float webcamAspect = (float)webcamTexture.width / (float)webcamTexture.height;
screen.rectTransform.sizeDelta = new Vector2(screen.rectTransform.sizeDelta.y * webcamAspect, (screen.rectTransform.sizeDelta.y));
screen.texture = webcamTexture;
// TODO this method is kinda meh you should use
_inputTexture = new Texture2D(_width, _height, TextureFormat.RGBA32, false);
_pixelData = new Color32[_width * _height];
inputTexture = new Texture2D(width, height, TextureFormat.RGBA32, false);
pixelData = new Color32[width * height];
if (!resourceManagerIsInitialized)
{
_resourceManager = new StreamingAssetsResourceManager();
yield return _resourceManager.PrepareAssetAsync("pose_detection.bytes");
yield return _resourceManager.PrepareAssetAsync("pose_landmark_full.bytes");
yield return _resourceManager.PrepareAssetAsync("face_landmark.bytes");
yield return _resourceManager.PrepareAssetAsync("hand_landmark_full.bytes");
yield return _resourceManager.PrepareAssetAsync("face_detection_short_range.bytes");
yield return _resourceManager.PrepareAssetAsync("hand_recrop.bytes");
yield return _resourceManager.PrepareAssetAsync("handedness.txt");
resourceManager = new StreamingAssetsResourceManager();
yield return resourceManager.PrepareAssetAsync("pose_detection.bytes");
yield return resourceManager.PrepareAssetAsync("pose_landmark_full.bytes");
yield return resourceManager.PrepareAssetAsync("face_landmark.bytes");
yield return resourceManager.PrepareAssetAsync("hand_landmark_full.bytes");
yield return resourceManager.PrepareAssetAsync("face_detection_short_range.bytes");
yield return resourceManager.PrepareAssetAsync("hand_recrop.bytes");
yield return resourceManager.PrepareAssetAsync("handedness.txt");
resourceManagerIsInitialized = true;
}
_stopwatch = new Stopwatch();
stopwatch = new Stopwatch();
// Setting up the graph
_graph = new CalculatorGraph(_configAsset.text);
graph = new CalculatorGraph(configAsset.text);
posestream = new OutputStream<NormalizedLandmarkListPacket, NormalizedLandmarkList>(_graph, "pose_landmarks", "pose_landmarks_presence");
leftstream = new OutputStream<NormalizedLandmarkListPacket, NormalizedLandmarkList>(_graph, "left_hand_landmarks", "left_hand_landmarks_presence");
rightstream = new OutputStream<NormalizedLandmarkListPacket, NormalizedLandmarkList>(_graph, "right_hand_landmarks", "right_hand_landmarks_presence");
posestream = new OutputStream<NormalizedLandmarkListPacket, NormalizedLandmarkList>(graph, "pose_landmarks", "pose_landmarks_presence");
leftstream = new OutputStream<NormalizedLandmarkListPacket, NormalizedLandmarkList>(graph, "left_hand_landmarks", "left_hand_landmarks_presence");
rightstream = new OutputStream<NormalizedLandmarkListPacket, NormalizedLandmarkList>(graph, "right_hand_landmarks", "right_hand_landmarks_presence");
posestream.StartPolling().AssertOk();
leftstream.StartPolling().AssertOk();
rightstream.StartPolling().AssertOk();
_graph.StartRun().AssertOk();
_stopwatch.Start();
graph.StartRun().AssertOk();
stopwatch.Start();
k = new KeypointManager();
keypointManager = new KeypointManager(modelInfoFile);
// check if model exists at path
var model = ModelLoader.Load(Resources.Load<NNModel>("Models/Fingerspelling/model_A-L"));
//var model = ModelLoader.Load(Resources.Load<NNModel>("Models/Fingerspelling/model_A-L"));
worker = model.CreateWorker();
StartCoroutine(SignRecognitionCoroutine());
@@ -211,25 +214,25 @@ namespace Mediapipe.Unity.Tutorial
{
while (true)
{
_inputTexture.SetPixels32(_webCamTexture.GetPixels32(_pixelData));
var imageFrame = new ImageFrame(ImageFormat.Types.Format.Srgba, _width, _height, _width * 4, _inputTexture.GetRawTextureData<byte>());
var currentTimestamp = _stopwatch.ElapsedTicks / (System.TimeSpan.TicksPerMillisecond / 1000);
_graph.AddPacketToInputStream("input_video", new ImageFramePacket(imageFrame, new Timestamp(currentTimestamp))).AssertOk();
inputTexture.SetPixels32(webcamTexture.GetPixels32(pixelData));
var imageFrame = new ImageFrame(ImageFormat.Types.Format.Srgba, width, height, width * 4, inputTexture.GetRawTextureData<byte>());
var currentTimestamp = stopwatch.ElapsedTicks / (System.TimeSpan.TicksPerMillisecond / 1000);
graph.AddPacketToInputStream("input_video", new ImageFramePacket(imageFrame, new Timestamp(currentTimestamp))).AssertOk();
//Debug.Log(Time.timeAsDouble + " Added new packet to mediapipe graph");
yield return new WaitForEndOfFrame();
Mediapipe.NormalizedLandmarkList _poseLandmarks = null;
Mediapipe.NormalizedLandmarkList _leftHandLandmarks = null;
Mediapipe.NormalizedLandmarkList _rightHandLandmarks = null;
NormalizedLandmarkList _poseLandmarks = null;
NormalizedLandmarkList _leftHandLandmarks = null;
NormalizedLandmarkList _rightHandLandmarks = null;
//Debug.Log("Extracting keypoints");
yield return new WaitUntil(() => { posestream.TryGetNext(out _poseLandmarks, false); return true;});
yield return new WaitUntil(() => { posestream.TryGetNext(out _poseLandmarks, false); return true; });
yield return new WaitUntil(() => { leftstream.TryGetNext(out _leftHandLandmarks, false); return true; });
yield return new WaitUntil(() => { rightstream.TryGetNext(out _rightHandLandmarks, false); return true; });
//Debug.Log(Time.timeAsDouble + " Retrieved landmarks ");
k.addLandmarks(_poseLandmarks, _leftHandLandmarks, _rightHandLandmarks);
keypointManager.AddLandmarks(_poseLandmarks, _leftHandLandmarks, _rightHandLandmarks);
}
}
@@ -241,7 +244,7 @@ namespace Mediapipe.Unity.Tutorial
{
while (true)
{
List<List<float>> input = k.getAllKeypoints();
List<List<float>> input = keypointManager.GetKeypoints();
if (input != null)
{
@@ -280,6 +283,7 @@ namespace Mediapipe.Unity.Tutorial
// Get the output as an array
float[] outputArray = output.ToReadOnlyArray();
//Debug.Log($"out = [{outputArray.Aggregate(" ", (t, f) => $"{t}{f} ")}]");
// Calculate the softmax of the output
float max = outputArray.Max();
@@ -295,16 +299,16 @@ namespace Mediapipe.Unity.Tutorial
float accuracy = (Mathf.RoundToInt(softmaxedOutput2[maxIndex] * 100));
// Set the letterProbabilities, currently used by Courses
letterProbabilities = new Dictionary<char, float>();
learnableProbabilities = new Dictionary<string, float>();
for (int i = 0; i < softmaxedOutput2.Length; i++)
{
letterProbabilities.Add((char)(i + 65), softmaxedOutput2[i]);
learnableProbabilities.Add(((char)(i + 65)).ToString(), softmaxedOutput2[i]);
}
//Debug.Log($"prob = [{learnableProbabilities.Aggregate(" ", (t, kv) => $"{t}{kv.Key}:{kv.Value} ")}]");
}
else
{
// Wait until next frame
//Debug.Log(Time.timeAsDouble + "No landmarks!");
yield return null;
}
}
@@ -315,29 +319,54 @@ namespace Mediapipe.Unity.Tutorial
/// </summary>
private void OnDestroy()
{
if (_webCamTexture != null)
if (webcamTexture != null)
{
_webCamTexture.Stop();
webcamTexture.Stop();
}
if (_graph != null)
if (graph != null)
{
try
{
_graph.CloseInputStream("input_video").AssertOk();
_graph.WaitUntilDone().AssertOk();
graph.CloseInputStream("input_video").AssertOk();
graph.WaitUntilDone().AssertOk();
}
finally
{
_graph.Dispose();
graph.Dispose();
}
}
// inputTensor must still be disposed, if it exists
inputTensor?.Dispose();
worker.Dispose();
worker?.Dispose();
}
/// <summary>
/// So long as there are cameras to use, you swap the camera you are using to another in the list.
/// </summary>
public void SwapCam()
{
if (WebCamTexture.devices.Length > 0)
{
// Stop the old camera
// If there was no camera playing before, then you dont have to reset the texture, as it wasn't assigned in the first place.
if (webcamTexture.isPlaying)
{
screen.texture = null;
webcamTexture.Stop();
webcamTexture = null;
}
// Find the new camera
camdex += 1;
camdex %= WebCamTexture.devices.Length;
// Start the new camera
WebCamDevice device = WebCamTexture.devices[camdex];
webcamTexture = new WebCamTexture(device.name);
screen.texture = webcamTexture;
webcamTexture.Play();
}
}
}
}

0
Assets/MediaPipeUnity/Scripts/Wesign_extractor.cs.meta → Assets/MediaPipeUnity/Scripts/SignPredictor.cs.meta
View File

0
Assets/MediaPipeUnity/WeSign_extractor_cpu.txt → Assets/MediaPipeUnity/SignPredictorCPU.txt
View File

0
Assets/MediaPipeUnity/WeSign_extractor_cpu.txt.meta → Assets/MediaPipeUnity/SignPredictorCPU.txt.meta
View File

0
Assets/MediaPipeUnity/WeSign_extractor_gpu.txt → Assets/MediaPipeUnity/SignPredictorGPU.txt
View File

0
Assets/MediaPipeUnity/WeSign_extractor_gpu.txt.meta → Assets/MediaPipeUnity/SignPredictorGPU.txt.meta
View File