using System.Collections; using System.Collections.Generic; using UnityEngine; using System.Linq; public class ModelInfo { public List pose_landmarks; public List hand_landmarks; } public class KeypointManager { private ModelInfo model_info; private List> keypoints_buffer; public KeypointManager() { TextAsset model_info_json = Resources.Load("Models/FingerSpelling/landmarks"); this.model_info = JsonUtility.FromJson(model_info_json.text); this.keypoints_buffer = new List>(); } private (List, List) normalizeHand(List hand_x, List hand_y) { float min_x = hand_x.Min(); float min_y = hand_y.Min(); float max_x = hand_x.Max(); float max_y = hand_y.Max(); float width = max_x - min_x; float height = max_y - min_y; if (width == 0 || height == 0) { return (hand_x, hand_y); } float center_x = (min_x + max_x) / 2; float center_y = (min_y + max_y) / 2; List normalized_x = new List(); List normalized_y = new List(); for (int i = 0; i < hand_x.Count; i++) { normalized_x.Add((hand_x[i] - center_x) / width); normalized_y.Add((hand_y[i] - center_y) / height); } return (normalized_x, normalized_y); } public void addLandmarks(Mediapipe.NormalizedLandmarkList poseLandmarks, Mediapipe.NormalizedLandmarkList leftHandLandmarks, Mediapipe.NormalizedLandmarkList rightHandLandmarks) { List pose_x = new List(); List pose_y = new List(); List left_hand_x = new List(); List left_hand_y = new List(); List right_hand_x = new List(); List right_hand_y = new List(); if (poseLandmarks != null) { foreach (var landmark_index in model_info.pose_landmarks) { pose_x.Add(poseLandmarks.Landmark[landmark_index].X); pose_y.Add(poseLandmarks.Landmark[landmark_index].Y); } } else { foreach (var landmark_index in model_info.pose_landmarks) { pose_x.Add(0); pose_y.Add(0); } } foreach (var landmark_index in model_info.hand_landmarks) { if (leftHandLandmarks == null) { left_hand_x.Add(0); left_hand_y.Add(0); } else { left_hand_x.Add(leftHandLandmarks.Landmark[landmark_index].X); left_hand_y.Add(leftHandLandmarks.Landmark[landmark_index].Y); } if (rightHandLandmarks == null) { right_hand_x.Add(0); right_hand_y.Add(0); } else { right_hand_x.Add(rightHandLandmarks.Landmark[landmark_index].X); right_hand_y.Add(rightHandLandmarks.Landmark[landmark_index].Y); } } // TODO: Add normalization (left_hand_x, left_hand_y) = normalizeHand(left_hand_x, left_hand_y); (right_hand_x, right_hand_y) = normalizeHand(right_hand_x, right_hand_y); if (keypoints_buffer.Count >= 10) { keypoints_buffer.RemoveAt(0); } List keypoints = new List(); for (int i = 0; i < pose_x.Count; i++) { keypoints.Add(pose_x[i]); keypoints.Add(pose_y[i]); } for (int i = 0; i < left_hand_x.Count; i++) { keypoints.Add(left_hand_x[i]); keypoints.Add(left_hand_y[i]); } for (int i = 0; i < right_hand_x.Count; i++) { keypoints.Add(right_hand_x[i]); keypoints.Add(right_hand_y[i]); } keypoints_buffer.Add(keypoints); } public List> getAllKeypoints() { if (keypoints_buffer.Count < 10) { return null; } return keypoints_buffer; } }