Wes xx mediapipe integration

Assets/MediaPipeUnity/Common/Scripts/ImageSourceSolution.cs

@@ -0,0 +1,131 @@
+// Copyright (c) 2021 homuler
+//
+// Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file or at
+// https://opensource.org/licenses/MIT.
+
+using System.Collections;
+using UnityEngine;
+
+namespace Mediapipe.Unity
+{
+  public abstract class ImageSourceSolution<T> : Solution where T : GraphRunner
+  {
+    [SerializeField] protected Screen screen;
+    [SerializeField] protected T graphRunner;
+    [SerializeField] protected TextureFramePool textureFramePool;
+
+    private Coroutine _coroutine;
+
+    public RunningMode runningMode;
+
+    public long timeoutMillisec
+    {
+      get => graphRunner.timeoutMillisec;
+      set => graphRunner.timeoutMillisec = value;
+    }
+
+    public override void Play()
+    {
+      if (_coroutine != null)
+      {
+        Stop();
+      }
+      base.Play();
+      _coroutine = StartCoroutine(Run());
+    }
+
+    public override void Pause()
+    {
+      base.Pause();
+      ImageSourceProvider.ImageSource.Pause();
+    }
+
+    public override void Resume()
+    {
+      base.Resume();
+      var _ = StartCoroutine(ImageSourceProvider.ImageSource.Resume());
+    }
+
+    public override void Stop()
+    {
+      base.Stop();
+      StopCoroutine(_coroutine);
+      ImageSourceProvider.ImageSource.Stop();
+      graphRunner.Stop();
+    }
+
+    private IEnumerator Run()
+    {
+      var graphInitRequest = graphRunner.WaitForInit(runningMode);
+      var imageSource = ImageSourceProvider.ImageSource;
+
+      yield return imageSource.Play();
+
+      if (!imageSource.isPrepared)
+      {
+        Logger.LogError(TAG, "Failed to start ImageSource, exiting...");
+        yield break;
+      }
+
+      // Use RGBA32 as the input format.
+      // TODO: When using GpuBuffer, MediaPipe assumes that the input format is BGRA, so the following code must be fixed.
+      textureFramePool.ResizeTexture(imageSource.textureWidth, imageSource.textureHeight, TextureFormat.RGBA32);
+      SetupScreen(imageSource);
+
+      yield return graphInitRequest;
+      if (graphInitRequest.isError)
+      {
+        Logger.LogError(TAG, graphInitRequest.error);
+        yield break;
+      }
+
+      OnStartRun();
+      graphRunner.StartRun(imageSource);
+
+      var waitWhilePausing = new WaitWhile(() => isPaused);
+
+      while (true)
+      {
+        if (isPaused)
+        {
+          yield return waitWhilePausing;
+        }
+
+        if (!textureFramePool.TryGetTextureFrame(out var textureFrame))
+        {
+          yield return new WaitForEndOfFrame();
+          continue;
+        }
+
+        // Copy current image to TextureFrame
+        ReadFromImageSource(imageSource, textureFrame);
+        AddTextureFrameToInputStream(textureFrame);
+        yield return new WaitForEndOfFrame();
+
+        if (runningMode.IsSynchronous())
+        {
+          RenderCurrentFrame(textureFrame);
+          yield return WaitForNextValue();
+        }
+      }
+    }
+
+    protected virtual void SetupScreen(ImageSource imageSource)
+    {
+      // NOTE: The screen will be resized later, keeping the aspect ratio.
+      screen.Initialize(imageSource);
+    }
+
+    protected virtual void RenderCurrentFrame(TextureFrame textureFrame)
+    {
+      screen.ReadSync(textureFrame);
+    }
+
+    protected abstract void OnStartRun();
+
+    protected abstract void AddTextureFrameToInputStream(TextureFrame textureFrame);
+
+    protected abstract IEnumerator WaitForNextValue();
+  }
+}