Wes xx mediapipe integration

Assets/MediaPipeUnity/Common/Scripts/ImageSource/ImageSource.cs

@@ -0,0 +1,159 @@
+// Copyright (c) 2021 homuler
+//
+// Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file or at
+// https://opensource.org/licenses/MIT.
+
+using System;
+using System.Collections;
+using UnityEngine;
+using UnityEngine.Experimental.Rendering;
+
+namespace Mediapipe.Unity
+{
+  public abstract class ImageSource : MonoBehaviour
+  {
+    [Serializable]
+    public struct ResolutionStruct
+    {
+      public int width;
+      public int height;
+      public double frameRate;
+
+      public ResolutionStruct(int width, int height, double frameRate)
+      {
+        this.width = width;
+        this.height = height;
+        this.frameRate = frameRate;
+      }
+
+      public ResolutionStruct(Resolution resolution)
+      {
+        width = resolution.width;
+        height = resolution.height;
+        frameRate = resolution.refreshRate;
+      }
+
+      public Resolution ToResolution()
+      {
+        return new Resolution() { width = width, height = height, refreshRate = (int)frameRate };
+      }
+
+      public override string ToString()
+      {
+        var aspectRatio = $"{width}x{height}";
+        var frameRateStr = frameRate.ToString("#.##");
+        return frameRate > 0 ? $"{aspectRatio} ({frameRateStr}Hz)" : aspectRatio;
+      }
+    }
+
+    public ResolutionStruct resolution { get; protected set; }
+
+    /// <remarks>
+    ///   To call this method, the image source must be prepared.
+    /// </remarks>
+    /// <returns>
+    ///   <see cref="TextureFormat" /> that is compatible with the current texture.
+    /// </returns>
+    public TextureFormat textureFormat => isPrepared ? TextureFormatFor(GetCurrentTexture()) : throw new InvalidOperationException("ImageSource is not prepared");
+    public virtual int textureWidth => resolution.width;
+    public virtual int textureHeight => resolution.height;
+    /// <remarks>
+    ///   If <see cref="type" /> does not support frame rate, it returns zero.
+    /// </remarks>
+    public virtual double frameRate => resolution.frameRate;
+    public float focalLengthPx { get; } = 2.0f; // TODO: calculate at runtime
+    public virtual bool isHorizontallyFlipped { get; set; } = false;
+    public virtual bool isVerticallyFlipped { get; } = false;
+    public virtual bool isFrontFacing { get; } = false;
+    public virtual RotationAngle rotation { get; } = RotationAngle.Rotation0;
+
+    public abstract string sourceName { get; }
+    public abstract string[] sourceCandidateNames { get; }
+    public abstract ResolutionStruct[] availableResolutions { get; }
+
+    /// <remarks>
+    ///   Once <see cref="Play" /> finishes successfully, it will become true.
+    /// </remarks>
+    /// <returns>
+    ///   Returns if the image source is prepared.
+    /// </returns>
+    public abstract bool isPrepared { get; }
+
+    public abstract bool isPlaying { get; }
+
+    /// <summary>
+    ///   Choose the source from <see cref="sourceCandidateNames" />.
+    /// </summary>
+    /// <remarks>
+    ///   You need to call <see cref="Play" /> for the change to take effect.
+    /// </remarks>
+    /// <param name="sourceId">The index of <see cref="sourceCandidateNames" /></param>
+    public abstract void SelectSource(int sourceId);
+
+    /// <summary>
+    ///   Choose the resolution from <see cref="availableResolutions" />.
+    /// </summary>
+    /// <remarks>
+    ///   You need to call <see cref="Play" /> for the change to take effect.
+    /// </remarks>
+    /// <param name="resolutionId">The index of <see cref="availableResolutions" /></param>
+    public void SelectResolution(int resolutionId)
+    {
+      var resolutions = availableResolutions;
+
+      if (resolutionId < 0 || resolutionId >= resolutions.Length)
+      {
+        throw new ArgumentException($"Invalid resolution ID: {resolutionId}");
+      }
+
+      resolution = resolutions[resolutionId];
+    }
+
+    /// <summary>
+    ///   Prepare image source.
+    ///   If <see cref="isPlaying" /> is true, it will reset the image source.
+    /// </summary>
+    /// <remarks>
+    ///   When it finishes successfully, <see cref="isPrepared" /> will return true.
+    /// </remarks>
+    /// <exception cref="InvalidOperation" />
+    public abstract IEnumerator Play();
+
+    /// <summary>
+    ///   Resume image source.
+    ///   If <see cref="isPlaying" /> is true, it will do nothing.
+    /// </summary>
+    /// <exception cref="InvalidOperation">
+    ///   The image source has not been played.
+    /// </exception>
+    public abstract IEnumerator Resume();
+
+    /// <summary>
+    ///   Pause image source.
+    ///   If <see cref="isPlaying" /> is false, it will do nothing.
+    /// </summary>
+    public abstract void Pause();
+
+    /// <summary>
+    ///   Stop image source.
+    /// </summary>
+    /// <remarks>
+    ///   When it finishes successfully, <see cref="isPrepared" /> will return false.
+    /// </remarks>
+    public abstract void Stop();
+
+    /// <remarks>
+    ///   To call this method, the image source must be prepared.
+    /// </remarks>
+    /// <returns>
+    ///   <see cref="Texture" /> that contains the current image.
+    /// </returns>
+    public abstract Texture GetCurrentTexture();
+
+    protected static TextureFormat TextureFormatFor(Texture texture)
+    {
+      return GraphicsFormatUtility.GetTextureFormat(texture.graphicsFormat);
+    }
+  }
+}

Assets/MediaPipeUnity/Common/Scripts/ImageSource/ImageSource.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: 78116299de071af7094419302302ec05
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Assets/MediaPipeUnity/Common/Scripts/ImageSource/ImageSourceType.cs

@@ -0,0 +1,13 @@
+// Copyright (c) 2021 homuler
+//
+// Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file or at
+// https://opensource.org/licenses/MIT.
+
+public enum ImageSourceType
+{
+  WebCamera = 0,
+  Image = 1,
+  Video = 2,
+  Unknown = 3,
+}

Assets/MediaPipeUnity/Common/Scripts/ImageSource/ImageSourceType.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: 2e20c591a3836aae1abc65429b7adde1
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Assets/MediaPipeUnity/Common/Scripts/ImageSource/StaticImageSource.cs

@@ -0,0 +1,136 @@
+// Copyright (c) 2021 homuler
+//
+// Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file or at
+// https://opensource.org/licenses/MIT.
+
+using System;
+using System.Collections;
+using System.Linq;
+using UnityEngine;
+
+namespace Mediapipe.Unity
+{
+  public class StaticImageSource : ImageSource
+  {
+    [SerializeField] private Texture[] _availableSources;
+
+    [SerializeField]
+    private ResolutionStruct[] _defaultAvailableResolutions = new ResolutionStruct[] {
+      new ResolutionStruct(512, 512, 0),
+      new ResolutionStruct(640, 480, 0),
+      new ResolutionStruct(1280, 720, 0),
+    };
+
+    private Texture2D _outputTexture;
+    private Texture _image;
+    private Texture image
+    {
+      get
+      {
+        if (_image == null && _availableSources != null && _availableSources.Length > 0)
+        {
+          image = _availableSources[0];
+        }
+        return _image;
+      }
+      set
+      {
+        _image = value;
+        resolution = GetDefaultResolution();
+      }
+    }
+
+    public override double frameRate => 0;
+
+    public override string sourceName => image != null ? image.name : null;
+
+    public override string[] sourceCandidateNames => _availableSources?.Select(source => source.name).ToArray();
+
+    public override ResolutionStruct[] availableResolutions => _defaultAvailableResolutions;
+
+    public override bool isPrepared => _outputTexture != null;
+
+    private bool _isPlaying = false;
+    public override bool isPlaying => _isPlaying;
+
+    public override void SelectSource(int sourceId)
+    {
+      if (sourceId < 0 || sourceId >= _availableSources.Length)
+      {
+        throw new ArgumentException($"Invalid source ID: {sourceId}");
+      }
+
+      image = _availableSources[sourceId];
+    }
+
+    public override IEnumerator Play()
+    {
+      if (image == null)
+      {
+        throw new InvalidOperationException("Image is not selected");
+      }
+      if (isPlaying)
+      {
+        yield break;
+      }
+
+      InitializeOutputTexture(image);
+      _isPlaying = true;
+      yield return null;
+    }
+
+    public override IEnumerator Resume()
+    {
+      if (!isPrepared)
+      {
+        throw new InvalidOperationException("Image is not prepared");
+      }
+      _isPlaying = true;
+
+      yield return null;
+    }
+
+    public override void Pause()
+    {
+      _isPlaying = false;
+    }
+    public override void Stop()
+    {
+      _isPlaying = false;
+      _outputTexture = null;
+    }
+
+    public override Texture GetCurrentTexture()
+    {
+      return _outputTexture;
+    }
+
+    private ResolutionStruct GetDefaultResolution()
+    {
+      var resolutions = availableResolutions;
+
+      return (resolutions == null || resolutions.Length == 0) ? new ResolutionStruct() : resolutions[0];
+    }
+
+    private void InitializeOutputTexture(Texture src)
+    {
+      _outputTexture = new Texture2D(textureWidth, textureHeight, TextureFormat.RGBA32, false);
+
+      Texture resizedTexture = new Texture2D(textureWidth, textureHeight, TextureFormat.RGBA32, false);
+      // TODO: assert ConvertTexture finishes successfully
+      var _ = Graphics.ConvertTexture(src, resizedTexture);
+
+      var currentRenderTexture = RenderTexture.active;
+      var tmpRenderTexture = new RenderTexture(resizedTexture.width, resizedTexture.height, 32);
+      Graphics.Blit(resizedTexture, tmpRenderTexture);
+      RenderTexture.active = tmpRenderTexture;
+
+      var rect = new UnityEngine.Rect(0, 0, _outputTexture.width, _outputTexture.height);
+      _outputTexture.ReadPixels(rect, 0, 0);
+      _outputTexture.Apply();
+
+      RenderTexture.active = currentRenderTexture;
+    }
+  }
+}

Assets/MediaPipeUnity/Common/Scripts/ImageSource/StaticImageSource.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: bd7955705ab46c72b9124bb116a2dca9
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Assets/MediaPipeUnity/Common/Scripts/ImageSource/TextureFrame.cs

@@ -0,0 +1,378 @@
+// Copyright (c) 2021 homuler
+//
+// Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file or at
+// https://opensource.org/licenses/MIT.
+
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using Unity.Collections;
+using UnityEngine;
+using UnityEngine.Events;
+using UnityEngine.Experimental.Rendering;
+
+namespace Mediapipe.Unity
+{
+#pragma warning disable IDE0065
+  using Color = UnityEngine.Color;
+#pragma warning restore IDE0065
+
+  public class TextureFrame
+  {
+    public class ReleaseEvent : UnityEvent<TextureFrame> { }
+
+    private const string _TAG = nameof(TextureFrame);
+
+    private static readonly GlobalInstanceTable<Guid, TextureFrame> _InstanceTable = new GlobalInstanceTable<Guid, TextureFrame>(100);
+    /// <summary>
+    ///   A dictionary to look up which native texture belongs to which <see cref="TextureFrame" />.
+    /// </summary>
+    /// <remarks>
+    ///   Not all the <see cref="TextureFrame" /> instances are registered.
+    ///   Texture names are queried only when necessary, and the corresponding data will be saved then.
+    /// </remarks>
+    private static readonly Dictionary<uint, Guid> _NameTable = new Dictionary<uint, Guid>();
+
+    private readonly Texture2D _texture;
+    private IntPtr _nativeTexturePtr = IntPtr.Zero;
+    private GlSyncPoint _glSyncToken;
+
+    // Buffers that will be used to copy texture data on CPU.
+    // They won't be initialized until it's necessary.
+    private Texture2D _textureBuffer;
+
+    private Color32[] _pixelsBuffer; // for WebCamTexture
+    private Color32[] pixelsBuffer
+    {
+      get
+      {
+        if (_pixelsBuffer == null)
+        {
+          _pixelsBuffer = new Color32[width * height];
+        }
+        return _pixelsBuffer;
+      }
+    }
+
+    private readonly Guid _instanceId;
+    // NOTE: width and height can be accessed from a thread other than Main Thread.
+    public readonly int width;
+    public readonly int height;
+    public readonly TextureFormat format;
+
+    private ImageFormat.Types.Format _format = ImageFormat.Types.Format.Unknown;
+    public ImageFormat.Types.Format imageFormat
+    {
+      get
+      {
+        if (_format == ImageFormat.Types.Format.Unknown)
+        {
+          _format = format.ToImageFormat();
+        }
+        return _format;
+      }
+    }
+
+    public bool isReadable => _texture.isReadable;
+
+    // TODO: determine at runtime
+    public GpuBufferFormat gpuBufferformat => GpuBufferFormat.kBGRA32;
+
+    /// <summary>
+    ///   The event that will be invoked when the TextureFrame is released.
+    /// </summary>
+#pragma warning disable IDE1006  // UnityEvent is PascalCase
+    public readonly ReleaseEvent OnRelease;
+#pragma warning restore IDE1006
+
+    private TextureFrame(Texture2D texture)
+    {
+      _texture = texture;
+      width = texture.width;
+      height = texture.height;
+      format = texture.format;
+      OnRelease = new ReleaseEvent();
+      _instanceId = Guid.NewGuid();
+      _InstanceTable.Add(_instanceId, this);
+    }
+
+    public TextureFrame(int width, int height, TextureFormat format) : this(new Texture2D(width, height, format, false)) { }
+    public TextureFrame(int width, int height) : this(width, height, TextureFormat.RGBA32) { }
+
+    public void CopyTexture(Texture dst)
+    {
+      Graphics.CopyTexture(_texture, dst);
+    }
+
+    public void CopyTextureFrom(Texture src)
+    {
+      Graphics.CopyTexture(src, _texture);
+    }
+
+    public bool ConvertTexture(Texture dst)
+    {
+      return Graphics.ConvertTexture(_texture, dst);
+    }
+
+    public bool ConvertTextureFrom(Texture src)
+    {
+      return Graphics.ConvertTexture(src, _texture);
+    }
+
+    /// <summary>
+    ///   Copy texture data from <paramref name="src" />.
+    ///   If <paramref name="src" /> format is different from <see cref="format" />, it converts the format.
+    /// </summary>
+    /// <remarks>
+    ///   After calling it, pixel data won't be read from CPU safely.
+    /// </remarks>
+    public bool ReadTextureFromOnGPU(Texture src)
+    {
+      if (GetTextureFormat(src) != format)
+      {
+        return Graphics.ConvertTexture(src, _texture);
+      }
+      Graphics.CopyTexture(src, _texture);
+      return true;
+    }
+
+    /// <summary>
+    ///   Copy texture data from <paramref name="src" />.
+    /// </summary>
+    /// <remarks>
+    ///   This operation is slow.
+    ///   If CPU won't access the pixel data, use <see cref="ReadTextureFromOnGPU" /> instead.
+    /// </remarks>
+    public void ReadTextureFromOnCPU(Texture src)
+    {
+      var textureBuffer = LoadToTextureBuffer(src);
+      SetPixels32(textureBuffer.GetPixels32());
+    }
+
+    /// <summary>
+    ///   Copy texture data from <paramref name="src" />.
+    /// </summary>
+    /// <remarks>
+    ///   In most cases, it should be better to use <paramref name="src" /> directly.
+    /// </remarks>
+    public void ReadTextureFromOnCPU(Texture2D src)
+    {
+      SetPixels32(src.GetPixels32());
+    }
+
+    /// <summary>
+    ///   Copy texture data from <paramref name="src" />.
+    /// </summary>
+    /// <param name="src">
+    ///   The texture from which the pixels are read.
+    ///   Its width and height must match that of the TextureFrame.
+    /// </param>
+    /// <remarks>
+    ///   This operation is slow.
+    ///   If CPU won't access the pixel data, use <see cref="ReadTextureFromOnGPU" /> instead.
+    /// </remarks>
+    public void ReadTextureFromOnCPU(WebCamTexture src)
+    {
+      SetPixels32(src.GetPixels32(pixelsBuffer));
+    }
+
+    public Color GetPixel(int x, int y)
+    {
+      return _texture.GetPixel(x, y);
+    }
+
+    public Color32[] GetPixels32()
+    {
+      return _texture.GetPixels32();
+    }
+
+    public void SetPixels32(Color32[] pixels)
+    {
+      _texture.SetPixels32(pixels);
+      _texture.Apply();
+
+      if (!RevokeNativeTexturePtr())
+      {
+        // If this line was executed, there must be a bug.
+        Logger.LogError("Failed to revoke the native texture.");
+      }
+    }
+
+    public NativeArray<T> GetRawTextureData<T>() where T : struct
+    {
+      return _texture.GetRawTextureData<T>();
+    }
+
+    /// <returns>The texture's native pointer</returns>
+    public IntPtr GetNativeTexturePtr()
+    {
+      if (_nativeTexturePtr == IntPtr.Zero)
+      {
+        _nativeTexturePtr = _texture.GetNativeTexturePtr();
+        var name = (uint)_nativeTexturePtr;
+
+        lock (((ICollection)_NameTable).SyncRoot)
+        {
+          if (!AcquireName(name, _instanceId))
+          {
+            throw new InvalidProgramException($"Another instance (id={_instanceId}) is using the specified name ({name}) now");
+          }
+          _NameTable.Add(name, _instanceId);
+        }
+      }
+      return _nativeTexturePtr;
+    }
+
+    public uint GetTextureName()
+    {
+      return (uint)GetNativeTexturePtr();
+    }
+
+    public Guid GetInstanceID()
+    {
+      return _instanceId;
+    }
+
+    public ImageFrame BuildImageFrame()
+    {
+      return new ImageFrame(imageFormat, width, height, 4 * width, GetRawTextureData<byte>());
+    }
+
+    public GpuBuffer BuildGpuBuffer(GlContext glContext)
+    {
+#if UNITY_EDITOR_LINUX || UNITY_STANDALONE_LINUX || UNITY_ANDROID
+      var glTextureBuffer = new GlTextureBuffer(GetTextureName(), width, height, gpuBufferformat, OnReleaseTextureFrame, glContext);
+      return new GpuBuffer(glTextureBuffer);
+#else
+      throw new NotSupportedException("This method is only supported on Linux or Android");
+#endif
+    }
+
+    public void RemoveAllReleaseListeners()
+    {
+      OnRelease.RemoveAllListeners();
+    }
+
+    // TODO: stop invoking OnRelease when it's already released
+    public void Release(GlSyncPoint token = null)
+    {
+      if (_glSyncToken != null)
+      {
+        _glSyncToken.Dispose();
+      }
+      _glSyncToken = token;
+      OnRelease.Invoke(this);
+    }
+
+    /// <summary>
+    ///   Waits until the GPU has executed all commands up to the sync point.
+    ///   This blocks the CPU, and ensures the commands are complete from the point of view of all threads and contexts.
+    /// </summary>
+    public void WaitUntilReleased()
+    {
+      if (_glSyncToken == null)
+      {
+        return;
+      }
+      _glSyncToken.Wait();
+      _glSyncToken.Dispose();
+      _glSyncToken = null;
+    }
+
+    [AOT.MonoPInvokeCallback(typeof(GlTextureBuffer.DeletionCallback))]
+    public static void OnReleaseTextureFrame(uint textureName, IntPtr syncTokenPtr)
+    {
+      var isIdFound = _NameTable.TryGetValue(textureName, out var _instanceId);
+
+      if (!isIdFound)
+      {
+        Logger.LogError(_TAG, $"nameof (name={textureName}) is released, but the owner TextureFrame is not found");
+        return;
+      }
+
+      var isTextureFrameFound = _InstanceTable.TryGetValue(_instanceId, out var textureFrame);
+
+      if (!isTextureFrameFound)
+      {
+        Logger.LogWarning(_TAG, $"nameof owner TextureFrame of the released texture (name={textureName}) is already garbage collected");
+        return;
+      }
+
+      var _glSyncToken = syncTokenPtr == IntPtr.Zero ? null : new GlSyncPoint(syncTokenPtr);
+      textureFrame.Release(_glSyncToken);
+    }
+
+    /// <summary>
+    ///   Remove <paramref name="name" /> from <see cref="_NameTable" /> if it's stale.
+    ///   If <paramref name="name" /> does not exist in <see cref="_NameTable" />, do nothing.
+    /// </summary>
+    /// <remarks>
+    ///   If the instance whose id is <paramref name="ownerId" /> owns <paramref name="name" /> now, it still removes <paramref name="name" />.
+    /// </remarks>
+    /// <returns>Return if name is available</returns>
+    private static bool AcquireName(uint name, Guid ownerId)
+    {
+      if (_NameTable.TryGetValue(name, out var id))
+      {
+        if (ownerId != id && _InstanceTable.TryGetValue(id, out var _))
+        {
+          // if instance is found, the instance is using the name.
+          Logger.LogVerbose($"{id} is using {name} now");
+          return false;
+        }
+        var _ = _NameTable.Remove(name);
+      }
+      return true;
+    }
+
+    private static TextureFormat GetTextureFormat(Texture texture)
+    {
+      return GraphicsFormatUtility.GetTextureFormat(texture.graphicsFormat);
+    }
+
+    /// <summary>
+    ///   Remove the texture name from <see cref="_NameTable" /> and empty <see cref="_nativeTexturePtr" />.
+    ///   This method needs to be called when an operation is performed that may change the internal texture.
+    /// </summary>
+    private bool RevokeNativeTexturePtr()
+    {
+      if (_nativeTexturePtr == IntPtr.Zero)
+      {
+        return true;
+      }
+
+      var currentName = GetTextureName();
+      if (!_NameTable.Remove(currentName))
+      {
+        return false;
+      }
+      _nativeTexturePtr = IntPtr.Zero;
+      return true;
+    }
+
+    private Texture2D LoadToTextureBuffer(Texture texture)
+    {
+      var textureFormat = GetTextureFormat(texture);
+
+      if (_textureBuffer == null || _textureBuffer.format != textureFormat)
+      {
+        _textureBuffer = new Texture2D(width, height, textureFormat, false);
+      }
+
+      var tmpRenderTexture = RenderTexture.GetTemporary(texture.width, texture.height, 32);
+      var currentRenderTexture = RenderTexture.active;
+      RenderTexture.active = tmpRenderTexture;
+
+      Graphics.Blit(texture, tmpRenderTexture);
+
+      var rect = new UnityEngine.Rect(0, 0, Mathf.Min(tmpRenderTexture.width, _textureBuffer.width), Mathf.Min(tmpRenderTexture.height, _textureBuffer.height));
+      _textureBuffer.ReadPixels(rect, 0, 0);
+      _textureBuffer.Apply();
+      RenderTexture.active = currentRenderTexture;
+      RenderTexture.ReleaseTemporary(tmpRenderTexture);
+
+      return _textureBuffer;
+    }
+  }
+}

Assets/MediaPipeUnity/Common/Scripts/ImageSource/TextureFrame.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: fa7e0da68d497cd578438e238b6a6e7c
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Assets/MediaPipeUnity/Common/Scripts/ImageSource/TextureFramePool.cs

@@ -0,0 +1,161 @@
+// Copyright (c) 2021 homuler
+//
+// Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file or at
+// https://opensource.org/licenses/MIT.
+
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using UnityEngine;
+
+namespace Mediapipe.Unity
+{
+  public class TextureFramePool : MonoBehaviour
+  {
+    private const string _TAG = nameof(TextureFramePool);
+
+    [SerializeField] private int _poolSize = 10;
+
+    private readonly object _formatLock = new object();
+    private int _textureWidth = 0;
+    private int _textureHeight = 0;
+    private TextureFormat _format = TextureFormat.RGBA32;
+
+    private Queue<TextureFrame> _availableTextureFrames;
+    /// <remarks>
+    ///   key: TextureFrame's instance ID
+    /// </remarks>
+    private Dictionary<Guid, TextureFrame> _textureFramesInUse;
+
+    /// <returns>
+    ///   The total number of texture frames in the pool.
+    /// </returns>
+    public int frameCount
+    {
+      get
+      {
+        var availableTextureFramesCount = _availableTextureFrames == null ? 0 : _availableTextureFrames.Count;
+        var textureFramesInUseCount = _textureFramesInUse == null ? 0 : _textureFramesInUse.Count;
+
+        return availableTextureFramesCount + textureFramesInUseCount;
+      }
+    }
+
+    private void Start()
+    {
+      _availableTextureFrames = new Queue<TextureFrame>(_poolSize);
+      _textureFramesInUse = new Dictionary<Guid, TextureFrame>();
+    }
+
+    private void OnDestroy()
+    {
+      lock (((ICollection)_availableTextureFrames).SyncRoot)
+      {
+        _availableTextureFrames.Clear();
+        _availableTextureFrames = null;
+      }
+
+      lock (((ICollection)_textureFramesInUse).SyncRoot)
+      {
+        foreach (var textureFrame in _textureFramesInUse.Values)
+        {
+          textureFrame.OnRelease.RemoveListener(OnTextureFrameRelease);
+        }
+        _textureFramesInUse.Clear();
+        _textureFramesInUse = null;
+      }
+    }
+
+    public void ResizeTexture(int textureWidth, int textureHeight, TextureFormat format)
+    {
+      lock (_formatLock)
+      {
+        _textureWidth = textureWidth;
+        _textureHeight = textureHeight;
+        _format = format;
+      }
+    }
+
+    public void ResizeTexture(int textureWidth, int textureHeight)
+    {
+      ResizeTexture(textureWidth, textureHeight, _format);
+    }
+
+    public bool TryGetTextureFrame(out TextureFrame outFrame)
+    {
+      TextureFrame nextFrame = null;
+
+      lock (((ICollection)_availableTextureFrames).SyncRoot)
+      {
+        if (_poolSize <= frameCount)
+        {
+          while (_availableTextureFrames.Count > 0)
+          {
+            var textureFrame = _availableTextureFrames.Dequeue();
+
+            if (!IsStale(textureFrame))
+            {
+              nextFrame = textureFrame;
+              break;
+            }
+          }
+        }
+        else
+        {
+          nextFrame = CreateNewTextureFrame();
+        }
+      }
+
+      if (nextFrame == null)
+      {
+        outFrame = null;
+        return false;
+      }
+
+      lock (((ICollection)_textureFramesInUse).SyncRoot)
+      {
+        _textureFramesInUse.Add(nextFrame.GetInstanceID(), nextFrame);
+      }
+
+      nextFrame.WaitUntilReleased();
+      outFrame = nextFrame;
+      return true;
+    }
+
+    private void OnTextureFrameRelease(TextureFrame textureFrame)
+    {
+      lock (((ICollection)_textureFramesInUse).SyncRoot)
+      {
+        if (!_textureFramesInUse.Remove(textureFrame.GetInstanceID()))
+        {
+          // won't be run
+          Logger.LogWarning(_TAG, "The released texture does not belong to the pool");
+          return;
+        }
+
+        if (frameCount > _poolSize || IsStale(textureFrame))
+        {
+          return;
+        }
+        _availableTextureFrames.Enqueue(textureFrame);
+      }
+    }
+
+    private bool IsStale(TextureFrame textureFrame)
+    {
+      lock (_formatLock)
+      {
+        return textureFrame.width != _textureWidth || textureFrame.height != _textureHeight;
+      }
+    }
+
+    private TextureFrame CreateNewTextureFrame()
+    {
+      var textureFrame = new TextureFrame(_textureWidth, _textureHeight, _format);
+      textureFrame.OnRelease.AddListener(OnTextureFrameRelease);
+
+      return textureFrame;
+    }
+  }
+}

Assets/MediaPipeUnity/Common/Scripts/ImageSource/TextureFramePool.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: d5da564da19cb6b7d8e4f97f269edc5d
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Assets/MediaPipeUnity/Common/Scripts/ImageSource/VideoSource.cs

@@ -0,0 +1,116 @@
+// Copyright (c) 2021 homuler
+//
+// Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file or at
+// https://opensource.org/licenses/MIT.
+
+using System;
+using System.Collections;
+using System.Linq;
+using UnityEngine;
+using UnityEngine.Video;
+
+namespace Mediapipe.Unity
+{
+  public class VideoSource : ImageSource
+  {
+    [SerializeField] private VideoClip[] _availableSources;
+
+    private VideoClip _video;
+    private VideoClip video
+    {
+      get
+      {
+        if (_video == null && _availableSources != null && _availableSources.Length > 0)
+        {
+          video = _availableSources[0];
+        }
+        return _video;
+      }
+      set
+      {
+        _video = value;
+        resolution = new ResolutionStruct((int)_video.width, (int)_video.height, _video.frameRate);
+      }
+    }
+
+    private VideoPlayer _videoPlayer;
+
+    public override string sourceName => video != null ? video.name : null;
+
+    public override string[] sourceCandidateNames => _availableSources?.Select(source => source.name).ToArray();
+
+    public override ResolutionStruct[] availableResolutions => video == null ? null : new ResolutionStruct[] { new ResolutionStruct((int)video.width, (int)video.height, video.frameRate) };
+
+    public override bool isPlaying => _videoPlayer != null && _videoPlayer.isPlaying;
+    public override bool isPrepared => _videoPlayer != null && _videoPlayer.isPrepared;
+
+    public override void SelectSource(int sourceId)
+    {
+      if (sourceId < 0 || sourceId >= _availableSources.Length)
+      {
+        throw new ArgumentException($"Invalid source ID: {sourceId}");
+      }
+
+      video = _availableSources[sourceId];
+      if (_videoPlayer != null)
+      {
+        _videoPlayer.clip = video;
+      }
+    }
+
+    public override IEnumerator Play()
+    {
+      if (video == null)
+      {
+        throw new InvalidOperationException("Video is not selected");
+      }
+      _videoPlayer = gameObject.AddComponent<VideoPlayer>();
+      _videoPlayer.renderMode = VideoRenderMode.APIOnly;
+      _videoPlayer.isLooping = true;
+      _videoPlayer.clip = video;
+      _videoPlayer.Prepare();
+
+      yield return new WaitUntil(() => _videoPlayer.isPrepared);
+      _videoPlayer.Play();
+    }
+
+    public override IEnumerator Resume()
+    {
+      if (!isPrepared)
+      {
+        throw new InvalidOperationException("VideoPlayer is not prepared");
+      }
+      if (!isPlaying)
+      {
+        _videoPlayer.Play();
+      }
+      yield return null;
+    }
+
+    public override void Pause()
+    {
+      if (!isPlaying)
+      {
+        return;
+      }
+      _videoPlayer.Pause();
+    }
+
+    public override void Stop()
+    {
+      if (_videoPlayer == null)
+      {
+        return;
+      }
+      _videoPlayer.Stop();
+      Destroy(gameObject.GetComponent<VideoPlayer>());
+      _videoPlayer = null;
+    }
+
+    public override Texture GetCurrentTexture()
+    {
+      return _videoPlayer != null ? _videoPlayer.texture : null;
+    }
+  }
+}

Assets/MediaPipeUnity/Common/Scripts/ImageSource/VideoSource.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: 04085488e5fac35599866a2a6fceeda3
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Assets/MediaPipeUnity/Common/Scripts/ImageSource/WebCamSource.cs

@@ -0,0 +1,299 @@
+// Copyright (c) 2021 homuler
+//
+// Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file or at
+// https://opensource.org/licenses/MIT.
+
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Linq;
+using UnityEngine;
+
+#if UNITY_ANDROID
+using UnityEngine.Android;
+#endif
+
+namespace Mediapipe.Unity
+{
+  public class WebCamSource : ImageSource
+  {
+    [Tooltip("For the default resolution, the one whose width is closest to this value will be chosen")]
+    [SerializeField] private int _preferableDefaultWidth = 1280;
+
+    private const string _TAG = nameof(WebCamSource);
+
+    [SerializeField]
+    private ResolutionStruct[] _defaultAvailableResolutions = new ResolutionStruct[] {
+      new ResolutionStruct(176, 144, 30),
+      new ResolutionStruct(320, 240, 30),
+      new ResolutionStruct(424, 240, 30),
+      new ResolutionStruct(640, 360, 30),
+      new ResolutionStruct(640, 480, 30),
+      new ResolutionStruct(848, 480, 30),
+      new ResolutionStruct(960, 540, 30),
+      new ResolutionStruct(1280, 720, 30),
+      new ResolutionStruct(1600, 896, 30),
+      new ResolutionStruct(1920, 1080, 30),
+    };
+
+    private static readonly object _PermissionLock = new object();
+    private static bool _IsPermitted = false;
+
+    private WebCamTexture _webCamTexture;
+    private WebCamTexture webCamTexture
+    {
+      get => _webCamTexture;
+      set
+      {
+        if (_webCamTexture != null)
+        {
+          _webCamTexture.Stop();
+        }
+        _webCamTexture = value;
+      }
+    }
+
+    public override int textureWidth => !isPrepared ? 0 : webCamTexture.width;
+    public override int textureHeight => !isPrepared ? 0 : webCamTexture.height;
+
+    public override bool isVerticallyFlipped => isPrepared && webCamTexture.videoVerticallyMirrored;
+    public override bool isFrontFacing => isPrepared && (webCamDevice is WebCamDevice valueOfWebCamDevice) && valueOfWebCamDevice.isFrontFacing;
+    public override RotationAngle rotation => !isPrepared ? RotationAngle.Rotation0 : (RotationAngle)webCamTexture.videoRotationAngle;
+
+    private WebCamDevice? _webCamDevice;
+    private WebCamDevice? webCamDevice
+    {
+      get => _webCamDevice;
+      set
+      {
+        if (_webCamDevice is WebCamDevice valueOfWebCamDevice)
+        {
+          if (value is WebCamDevice valueOfValue && valueOfValue.name == valueOfWebCamDevice.name)
+          {
+            // not changed
+            return;
+          }
+        }
+        else if (value == null)
+        {
+          // not changed
+          return;
+        }
+        _webCamDevice = value;
+        resolution = GetDefaultResolution();
+      }
+    }
+    public override string sourceName => (webCamDevice is WebCamDevice valueOfWebCamDevice) ? valueOfWebCamDevice.name : null;
+
+    private WebCamDevice[] _availableSources;
+    private WebCamDevice[] availableSources
+    {
+      get
+      {
+        if (_availableSources == null)
+        {
+          _availableSources = WebCamTexture.devices;
+        }
+
+        return _availableSources;
+      }
+      set => _availableSources = value;
+    }
+
+    public override string[] sourceCandidateNames => availableSources?.Select(device => device.name).ToArray();
+
+#pragma warning disable IDE0025
+    public override ResolutionStruct[] availableResolutions
+    {
+      get
+      {
+#if (UNITY_ANDROID || UNITY_IOS) && !UNITY_EDITOR
+        if (webCamDevice is WebCamDevice valueOfWebCamDevice) {
+          return valueOfWebCamDevice.availableResolutions.Select(resolution => new ResolutionStruct(resolution)).ToArray();
+        }
+#endif
+        return webCamDevice == null ? null : _defaultAvailableResolutions;
+      }
+    }
+#pragma warning restore IDE0025
+
+    public override bool isPrepared => webCamTexture != null;
+    public override bool isPlaying => webCamTexture != null && webCamTexture.isPlaying;
+    private bool _isInitialized;
+
+    private IEnumerator Start()
+    {
+      yield return GetPermission();
+
+      if (!_IsPermitted)
+      {
+        _isInitialized = true;
+        yield break;
+      }
+
+      availableSources = WebCamTexture.devices;
+
+      if (availableSources != null && availableSources.Length > 0)
+      {
+        webCamDevice = availableSources[0];
+      }
+
+      _isInitialized = true;
+    }
+
+    private IEnumerator GetPermission()
+    {
+      lock (_PermissionLock)
+      {
+        if (_IsPermitted)
+        {
+          yield break;
+        }
+
+#if UNITY_ANDROID
+        if (!Permission.HasUserAuthorizedPermission(Permission.Camera))
+        {
+          Permission.RequestUserPermission(Permission.Camera);
+          yield return new WaitForSeconds(0.1f);
+        }
+#elif UNITY_IOS
+        if (!Application.HasUserAuthorization(UserAuthorization.WebCam)) {
+          yield return Application.RequestUserAuthorization(UserAuthorization.WebCam);
+        }
+#endif
+
+#if UNITY_ANDROID
+        if (!Permission.HasUserAuthorizedPermission(Permission.Camera))
+        {
+          Logger.LogWarning(_TAG, "Not permitted to use Camera");
+          yield break;
+        }
+#elif UNITY_IOS
+        if (!Application.HasUserAuthorization(UserAuthorization.WebCam)) {
+          Logger.LogWarning(_TAG, "Not permitted to use WebCam");
+          yield break;
+        }
+#endif
+        _IsPermitted = true;
+
+        yield return new WaitForEndOfFrame();
+      }
+    }
+
+    public override void SelectSource(int sourceId)
+    {
+      if (sourceId < 0 || sourceId >= availableSources.Length)
+      {
+        throw new ArgumentException($"Invalid source ID: {sourceId}");
+      }
+
+      webCamDevice = availableSources[sourceId];
+    }
+
+    public override IEnumerator Play()
+    {
+      yield return new WaitUntil(() => _isInitialized);
+      if (!_IsPermitted)
+      {
+        throw new InvalidOperationException("Not permitted to access cameras");
+      }
+
+      InitializeWebCamTexture();
+      webCamTexture.Play();
+      yield return WaitForWebCamTexture();
+    }
+
+    public override IEnumerator Resume()
+    {
+      if (!isPrepared)
+      {
+        throw new InvalidOperationException("WebCamTexture is not prepared yet");
+      }
+      if (!webCamTexture.isPlaying)
+      {
+        webCamTexture.Play();
+      }
+      yield return WaitForWebCamTexture();
+    }
+
+    public override void Pause()
+    {
+      if (isPlaying)
+      {
+        webCamTexture.Pause();
+      }
+    }
+
+    public override void Stop()
+    {
+      if (webCamTexture != null)
+      {
+        webCamTexture.Stop();
+      }
+      webCamTexture = null;
+    }
+
+    public override Texture GetCurrentTexture()
+    {
+      return webCamTexture;
+    }
+
+    private ResolutionStruct GetDefaultResolution()
+    {
+      var resolutions = availableResolutions;
+      return resolutions == null || resolutions.Length == 0 ? new ResolutionStruct() : resolutions.OrderBy(resolution => resolution, new ResolutionStructComparer(_preferableDefaultWidth)).First();
+    }
+
+    private void InitializeWebCamTexture()
+    {
+      Stop();
+      if (webCamDevice is WebCamDevice valueOfWebCamDevice)
+      {
+        webCamTexture = new WebCamTexture(valueOfWebCamDevice.name, resolution.width, resolution.height, (int)resolution.frameRate);
+        return;
+      }
+      throw new InvalidOperationException("Cannot initialize WebCamTexture because WebCamDevice is not selected");
+    }
+
+    private IEnumerator WaitForWebCamTexture()
+    {
+      const int timeoutFrame = 2000;
+      var count = 0;
+      Logger.LogVerbose("Waiting for WebCamTexture to start");
+      yield return new WaitUntil(() => count++ > timeoutFrame || webCamTexture.width > 16);
+
+      if (webCamTexture.width <= 16)
+      {
+        throw new TimeoutException("Failed to start WebCam");
+      }
+    }
+
+    private class ResolutionStructComparer : IComparer<ResolutionStruct>
+    {
+      private readonly int _preferableDefaultWidth;
+
+      public ResolutionStructComparer(int preferableDefaultWidth)
+      {
+        _preferableDefaultWidth = preferableDefaultWidth;
+      }
+
+      public int Compare(ResolutionStruct a, ResolutionStruct b)
+      {
+        var aDiff = Mathf.Abs(a.width - _preferableDefaultWidth);
+        var bDiff = Mathf.Abs(b.width - _preferableDefaultWidth);
+        if (aDiff != bDiff)
+        {
+          return aDiff - bDiff;
+        }
+        if (a.height != b.height)
+        {
+          // prefer smaller height
+          return a.height - b.height;
+        }
+        // prefer smaller frame rate
+        return (int)(a.frameRate - b.frameRate);
+      }
+    }
+  }
+}

Assets/MediaPipeUnity/Common/Scripts/ImageSource/WebCamSource.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: 498146e99d4934673bd948c8be11227e
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: