using Mediapipe; using System.Collections.Generic; using System.Linq; using UnityEngine; public class KeypointManagerEmbedding { private int leftShoulderIndex = 11; private int rightShoulderIndex = 12; private int neckIndex = 33; private int noseIndex = 0; private int leftEyeIndex = 2; private List pose_indices = new List { 0, 33, 5, 2, 8, 7, 12, 11, 14, 13, 16, 15 }; private List hand_indices = new List { 0, 8, 7, 6, 5, 12, 11, 10, 9, 16, 15, 14, 13, 20, 19, 18, 17, 4, 3, 2, 1 }; private static int BUFFER_SIZE = 10; private List>> keypointsBuffer; public KeypointManagerEmbedding() { keypointsBuffer = new List>>(); } private (List, List) NormalizeHand(List handX, List handY) { var xValues = new List(); var yValues = new List(); for (int i = 0; i < handX.Count; i++) { if (handX[i] != 0) { xValues.Add(handX[i]); } if (handY[i] != 0) { yValues.Add(handY[i]); } } if (xValues.Count == 0 || yValues.Count == 0) { return (handX, handY); } float width = xValues.Max() - xValues.Min(); float height = yValues.Max() - yValues.Min(); float delta_x, delta_y; if (width > height) { delta_x = 0.1f * width; delta_y = delta_x + ((width - height) / 2f); } else { delta_y = 0.1f * height; delta_x = delta_y + ((height - width) / 2f); } var startingPoint = new Vector2(xValues.Min() - delta_x, yValues.Min() - delta_y); var endingPoint = new Vector2(xValues.Max() + delta_x, yValues.Max() + delta_y); if (endingPoint.x - startingPoint.x == 0f || endingPoint.y - startingPoint.y == 0f) { return (handX, handY); } // normalize keypoints for (int i = 0; i < handX.Count; i++) { handX[i] = (handX[i] - startingPoint.x) / (endingPoint.x - startingPoint.x); handY[i] = (handY[i] - startingPoint.y) / (endingPoint.y - startingPoint.y); } return (handX, handY); } private (List, List) NormalizePose(List poseX, List poseY) { var leftShoulder = new Vector2(poseX[leftShoulderIndex], poseY[leftShoulderIndex]); var rightShoulder = new Vector2(poseX[rightShoulderIndex], poseY[rightShoulderIndex]); var neck = new Vector2(poseX[neckIndex], poseY[neckIndex]); var nose = new Vector2(poseX[noseIndex], poseY[noseIndex]); // Prevent from even starting the analysis if some necessary elements are not present if ((leftShoulder.x == 0 || rightShoulder.x == 0 || (leftShoulder.x == rightShoulder.x && leftShoulder.y == rightShoulder.y)) && (neck.x == 0 || nose.x == 0 || (neck.x == nose.x && neck.y == nose.y))) { return (poseX, poseY); } float shoulderDistance, headMetric; if (leftShoulder.x != 0 && rightShoulder.x != 0 && (leftShoulder.x != rightShoulder.x || leftShoulder.y != rightShoulder.y)) { shoulderDistance = Mathf.Sqrt(Mathf.Pow(leftShoulder.x - rightShoulder.x, 2) + Mathf.Pow(leftShoulder.y - rightShoulder.y, 2)); headMetric = shoulderDistance; } else { float neckNoseDistance = Mathf.Sqrt(Mathf.Pow(neck.x - nose.x, 2) + Mathf.Pow(neck.y - nose.y, 2)); headMetric = neckNoseDistance; } // Set the starting and ending point of the normalization bounding box var startingPoint = new Vector2(poseX[neckIndex] - 3 * headMetric, poseY[leftEyeIndex] + headMetric); var endingPoint = new Vector2(poseX[neckIndex] + 3 * headMetric, startingPoint.y - 6 * headMetric); if (startingPoint.x < 0) { startingPoint.x = 0; } if (startingPoint.y < 0) { startingPoint.y = 0; } if (endingPoint.x < 0) { endingPoint.x = 0; } if (endingPoint.y < 0) { endingPoint.y = 0; } // Normalize the keypoints for (int i = 0; i < poseX.Count; i++) { poseX[i] = (poseX[i] - startingPoint.x) / (endingPoint.x - startingPoint.x); poseY[i] = (poseY[i] - endingPoint.y) / (startingPoint.y - endingPoint.y); } return (poseX, poseY); } private (List, List) CalculateNeck(List keypointsX, List keypointsY) { var leftShoulder = new Vector2(keypointsX[11], keypointsY[11]); var rightShoulder = new Vector2(keypointsX[12], keypointsY[12]); var neck = new Vector2((leftShoulder.x + rightShoulder.x) / 2, (leftShoulder.y + rightShoulder.y) / 2); // add neck to keypoints keypointsX.Add(neck.x); keypointsY.Add(neck.y); return (keypointsX, keypointsY); } public void AddLandmarks(NormalizedLandmarkList poseLandmarks, NormalizedLandmarkList leftHandLandmarks, NormalizedLandmarkList rightHandLandmarks) { List pose_x = new List(); List pose_y = new List(); List left_hand_x = new List(); List left_hand_y = new List(); List right_hand_x = new List(); List right_hand_y = new List(); if (poseLandmarks == null || (leftHandLandmarks == null && rightHandLandmarks == null)) { return; } if (poseLandmarks != null) { foreach (NormalizedLandmark landmark in poseLandmarks.Landmark) { pose_x.Add(landmark.X); pose_y.Add(landmark.Y); } }else{ for (int i = 0; i < 33; i++) { pose_x.Add(0); pose_y.Add(0); } } // Add neck to pose (pose_x, pose_y) = CalculateNeck(pose_x, pose_y); // normalize pose (pose_x, pose_y) = NormalizePose(pose_x, pose_y); // now filter the pose keypoints based on the pose indeces List> filtered_pose = new List>(); foreach (int index in pose_indices) { filtered_pose.Add(new List { pose_x[index] - 0.5f, pose_y[index] - 0.5f }); } // add hand landmarks if (leftHandLandmarks != null) { foreach (NormalizedLandmark landmark in leftHandLandmarks.Landmark) { left_hand_x.Add(landmark.X); left_hand_y.Add(landmark.Y); } }else{ for (int i = 0; i < 21; i++) { left_hand_x.Add(0); left_hand_y.Add(0); } } if (rightHandLandmarks != null) { foreach (NormalizedLandmark landmark in rightHandLandmarks.Landmark) { right_hand_x.Add(landmark.X); right_hand_y.Add(landmark.Y); } }else{ for (int i = 0; i < 21; i++) { right_hand_x.Add(0); right_hand_y.Add(0); } } // normalize the hands (left_hand_x, left_hand_y) = NormalizeHand(left_hand_x, left_hand_y); (right_hand_x, right_hand_y) = NormalizeHand(right_hand_x, right_hand_y); // now filter the hand keypoints based on the hand indeces List> filtered_left_hand = new List>(); List> filtered_right_hand = new List>(); foreach (int index in hand_indices) { filtered_left_hand.Add(new List { left_hand_x[index] - 0.5f, left_hand_y[index] - 0.5f }); filtered_right_hand.Add(new List { right_hand_x[index] - 0.5f, right_hand_y[index] - 0.5f }); } // add the filtered keypoints together in one list List> filtered_keypoints = new List>(); filtered_keypoints.AddRange(filtered_pose); filtered_keypoints.AddRange(filtered_left_hand); filtered_keypoints.AddRange(filtered_right_hand); keypointsBuffer.Add(filtered_keypoints); if (keypointsBuffer.Count > BUFFER_SIZE) { keypointsBuffer.RemoveAt(0); } } public List>> GetKeypoints() { if (keypointsBuffer.Count < BUFFER_SIZE){ return null; } return keypointsBuffer; } }