Wes xx mediapipe integration
This commit is contained in:
39
Assets/MediaPipeUnity/Common/Scripts/AssetLoader.cs
Normal file
39
Assets/MediaPipeUnity/Common/Scripts/AssetLoader.cs
Normal file
@@ -0,0 +1,39 @@
|
||||
// Copyright (c) 2021 homuler
|
||||
//
|
||||
// Use of this source code is governed by an MIT-style
|
||||
// license that can be found in the LICENSE file or at
|
||||
// https://opensource.org/licenses/MIT.
|
||||
|
||||
using System.Collections;
|
||||
|
||||
namespace Mediapipe.Unity
|
||||
{
|
||||
public static class AssetLoader
|
||||
{
|
||||
private static ResourceManager _ResourceManager;
|
||||
|
||||
public static void Provide(ResourceManager manager)
|
||||
{
|
||||
_ResourceManager = manager;
|
||||
}
|
||||
|
||||
public static IEnumerator PrepareAssetAsync(string name, string uniqueKey, bool overwrite = false)
|
||||
{
|
||||
if (_ResourceManager == null)
|
||||
{
|
||||
#if UNITY_EDITOR
|
||||
Logger.LogWarning("ResourceManager is not provided, so default LocalResourceManager will be used");
|
||||
_ResourceManager = new LocalResourceManager();
|
||||
#else
|
||||
throw new System.InvalidOperationException("ResourceManager is not provided");
|
||||
#endif
|
||||
}
|
||||
return _ResourceManager.PrepareAssetAsync(name, uniqueKey, overwrite);
|
||||
}
|
||||
|
||||
public static IEnumerator PrepareAssetAsync(string name, bool overwrite = false)
|
||||
{
|
||||
return PrepareAssetAsync(name, name, overwrite);
|
||||
}
|
||||
}
|
||||
}
|
||||
11
Assets/MediaPipeUnity/Common/Scripts/AssetLoader.cs.meta
Normal file
11
Assets/MediaPipeUnity/Common/Scripts/AssetLoader.cs.meta
Normal file
@@ -0,0 +1,11 @@
|
||||
fileFormatVersion: 2
|
||||
guid: b8bd1b384b7c7414cb732e8fbabf95a8
|
||||
MonoImporter:
|
||||
externalObjects: {}
|
||||
serializedVersion: 2
|
||||
defaultReferences: []
|
||||
executionOrder: 0
|
||||
icon: {instanceID: 0}
|
||||
userData:
|
||||
assetBundleName:
|
||||
assetBundleVariant:
|
||||
160
Assets/MediaPipeUnity/Common/Scripts/Bootstrap.cs
Normal file
160
Assets/MediaPipeUnity/Common/Scripts/Bootstrap.cs
Normal file
@@ -0,0 +1,160 @@
|
||||
// Copyright (c) 2021 homuler
|
||||
//
|
||||
// Use of this source code is governed by an MIT-style
|
||||
// license that can be found in the LICENSE file or at
|
||||
// https://opensource.org/licenses/MIT.
|
||||
|
||||
using System.Collections;
|
||||
using System.IO;
|
||||
using UnityEngine;
|
||||
|
||||
namespace Mediapipe.Unity
|
||||
{
|
||||
public class Bootstrap : MonoBehaviour
|
||||
{
|
||||
[System.Serializable]
|
||||
public enum AssetLoaderType
|
||||
{
|
||||
StreamingAssets,
|
||||
AssetBundle,
|
||||
Local,
|
||||
}
|
||||
|
||||
private const string _TAG = nameof(Bootstrap);
|
||||
|
||||
[SerializeField] private ImageSourceType _defaultImageSource;
|
||||
[SerializeField] private InferenceMode _preferableInferenceMode;
|
||||
[SerializeField] private AssetLoaderType _assetLoaderType;
|
||||
[SerializeField] private bool _enableGlog = true;
|
||||
|
||||
public InferenceMode inferenceMode { get; private set; }
|
||||
public bool isFinished { get; private set; }
|
||||
private bool _isGlogInitialized;
|
||||
|
||||
private void OnEnable()
|
||||
{
|
||||
var _ = StartCoroutine(Init());
|
||||
}
|
||||
|
||||
private IEnumerator Init()
|
||||
{
|
||||
Logger.SetLogger(new MemoizedLogger(100));
|
||||
Logger.MinLogLevel = Logger.LogLevel.Debug;
|
||||
|
||||
Protobuf.SetLogHandler(Protobuf.DefaultLogHandler);
|
||||
|
||||
Logger.LogInfo(_TAG, "Setting global flags...");
|
||||
GlobalConfigManager.SetFlags();
|
||||
|
||||
if (_enableGlog)
|
||||
{
|
||||
if (Glog.LogDir != null)
|
||||
{
|
||||
if (!Directory.Exists(Glog.LogDir))
|
||||
{
|
||||
Directory.CreateDirectory(Glog.LogDir);
|
||||
}
|
||||
Logger.LogVerbose(_TAG, $"Glog will output files under {Glog.LogDir}");
|
||||
}
|
||||
Glog.Initialize("MediaPipeUnityPlugin");
|
||||
_isGlogInitialized = true;
|
||||
}
|
||||
|
||||
Logger.LogInfo(_TAG, "Initializing AssetLoader...");
|
||||
switch (_assetLoaderType)
|
||||
{
|
||||
case AssetLoaderType.AssetBundle:
|
||||
{
|
||||
AssetLoader.Provide(new AssetBundleResourceManager("mediapipe"));
|
||||
break;
|
||||
}
|
||||
case AssetLoaderType.StreamingAssets:
|
||||
{
|
||||
AssetLoader.Provide(new StreamingAssetsResourceManager());
|
||||
break;
|
||||
}
|
||||
case AssetLoaderType.Local:
|
||||
{
|
||||
#if UNITY_EDITOR
|
||||
AssetLoader.Provide(new LocalResourceManager());
|
||||
break;
|
||||
#else
|
||||
Logger.LogError("LocalResourceManager is only supported on UnityEditor");
|
||||
yield break;
|
||||
#endif
|
||||
}
|
||||
default:
|
||||
{
|
||||
Logger.LogError($"AssetLoaderType is unknown: {_assetLoaderType}");
|
||||
yield break;
|
||||
}
|
||||
}
|
||||
|
||||
DecideInferenceMode();
|
||||
if (inferenceMode == InferenceMode.GPU)
|
||||
{
|
||||
Logger.LogInfo(_TAG, "Initializing GPU resources...");
|
||||
yield return GpuManager.Initialize();
|
||||
|
||||
if (!GpuManager.IsInitialized)
|
||||
{
|
||||
Logger.LogWarning("If your native library is built for CPU, change 'Preferable Inference Mode' to CPU from the Inspector Window for Bootstrap");
|
||||
}
|
||||
}
|
||||
|
||||
Logger.LogInfo(_TAG, "Preparing ImageSource...");
|
||||
ImageSourceProvider.ImageSource = GetImageSource(_defaultImageSource);
|
||||
|
||||
isFinished = true;
|
||||
}
|
||||
|
||||
public ImageSource GetImageSource(ImageSourceType imageSourceType)
|
||||
{
|
||||
switch (imageSourceType)
|
||||
{
|
||||
case ImageSourceType.WebCamera:
|
||||
{
|
||||
return GetComponent<WebCamSource>();
|
||||
}
|
||||
case ImageSourceType.Image:
|
||||
{
|
||||
return GetComponent<StaticImageSource>();
|
||||
}
|
||||
case ImageSourceType.Video:
|
||||
{
|
||||
return GetComponent<VideoSource>();
|
||||
}
|
||||
case ImageSourceType.Unknown:
|
||||
default:
|
||||
{
|
||||
throw new System.ArgumentException($"Unsupported source type: {imageSourceType}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void DecideInferenceMode()
|
||||
{
|
||||
#if UNITY_EDITOR_OSX || UNITY_EDITOR_WIN
|
||||
if (_preferableInferenceMode == InferenceMode.GPU) {
|
||||
Logger.LogWarning(_TAG, "Current platform does not support GPU inference mode, so falling back to CPU mode");
|
||||
}
|
||||
inferenceMode = InferenceMode.CPU;
|
||||
#else
|
||||
inferenceMode = _preferableInferenceMode;
|
||||
#endif
|
||||
}
|
||||
|
||||
private void OnApplicationQuit()
|
||||
{
|
||||
GpuManager.Shutdown();
|
||||
|
||||
if (_isGlogInitialized)
|
||||
{
|
||||
Glog.Shutdown();
|
||||
}
|
||||
|
||||
Protobuf.ResetLogHandler();
|
||||
Logger.SetLogger(null);
|
||||
}
|
||||
}
|
||||
}
|
||||
15
Assets/MediaPipeUnity/Common/Scripts/Bootstrap.cs.meta
Normal file
15
Assets/MediaPipeUnity/Common/Scripts/Bootstrap.cs.meta
Normal file
@@ -0,0 +1,15 @@
|
||||
fileFormatVersion: 2
|
||||
guid: f4d846393f8d9f20fa64b924b0d95e68
|
||||
MonoImporter:
|
||||
externalObjects: {}
|
||||
serializedVersion: 2
|
||||
defaultReferences:
|
||||
- globalConfigManagerPrefab: {fileID: 4394522812020757894, guid: 488fbb6a3005b66c2b657204bce83fcf,
|
||||
type: 3}
|
||||
- gpuManagerPrefab: {fileID: 7458244645391269047, guid: a84904e562ecaf887b2dba62111bb901,
|
||||
type: 3}
|
||||
executionOrder: 0
|
||||
icon: {instanceID: 0}
|
||||
userData:
|
||||
assetBundleName:
|
||||
assetBundleVariant:
|
||||
140
Assets/MediaPipeUnity/Common/Scripts/GlobalConfigManager.cs
Normal file
140
Assets/MediaPipeUnity/Common/Scripts/GlobalConfigManager.cs
Normal file
@@ -0,0 +1,140 @@
|
||||
// Copyright (c) 2021 homuler
|
||||
//
|
||||
// Use of this source code is governed by an MIT-style
|
||||
// license that can be found in the LICENSE file or at
|
||||
// https://opensource.org/licenses/MIT.
|
||||
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Text;
|
||||
using UnityEngine;
|
||||
|
||||
namespace Mediapipe.Unity
|
||||
{
|
||||
public static class GlobalConfigManager
|
||||
{
|
||||
private const string _TAG = nameof(GlobalConfigManager);
|
||||
|
||||
private static string CacheDirPath => Path.Combine(Application.persistentDataPath, "Cache");
|
||||
|
||||
private static string ConfigFilePath => Path.Combine(CacheDirPath, "globalConfig.env");
|
||||
|
||||
private const string _GlogLogtostderrKey = "GLOG_logtostderr";
|
||||
private const string _GlogStderrthresholdKey = "GLOG_stderrthreshold";
|
||||
private const string _GlogMinloglevelKey = "GLOG_minloglevel";
|
||||
private const string _GlogVKey = "GLOG_v";
|
||||
private const string _GlogLogDirKey = "GLOG_log_dir";
|
||||
|
||||
public static bool GlogLogtostderr
|
||||
{
|
||||
get => Config[_GlogLogtostderrKey] == "1";
|
||||
set => Config[_GlogLogtostderrKey] = value ? "1" : "0";
|
||||
}
|
||||
|
||||
public static int GlogStderrthreshold
|
||||
{
|
||||
get => int.Parse(Config[_GlogStderrthresholdKey]);
|
||||
set => Config[_GlogStderrthresholdKey] = value.ToString();
|
||||
}
|
||||
|
||||
public static int GlogMinloglevel
|
||||
{
|
||||
get => int.Parse(Config[_GlogMinloglevelKey]);
|
||||
set => Config[_GlogMinloglevelKey] = value.ToString();
|
||||
}
|
||||
|
||||
public static int GlogV
|
||||
{
|
||||
get => int.Parse(Config[_GlogVKey]);
|
||||
set => Config[_GlogVKey] = value.ToString();
|
||||
}
|
||||
|
||||
public static string GlogLogDir
|
||||
{
|
||||
get => Config[_GlogLogDirKey];
|
||||
set => Config[_GlogLogDirKey] = value;
|
||||
}
|
||||
|
||||
private static Dictionary<string, string> _Config;
|
||||
private static Dictionary<string, string> Config
|
||||
{
|
||||
get
|
||||
{
|
||||
if (_Config == null)
|
||||
{
|
||||
_Config = new Dictionary<string, string>() {
|
||||
{ _GlogLogtostderrKey, "1" },
|
||||
{ _GlogStderrthresholdKey, "2" },
|
||||
{ _GlogMinloglevelKey, "0" },
|
||||
{ _GlogLogDirKey, "" },
|
||||
{ _GlogVKey, "0" },
|
||||
};
|
||||
|
||||
if (!File.Exists(ConfigFilePath))
|
||||
{
|
||||
Logger.LogDebug(_TAG, $"Global config file does not exist: {ConfigFilePath}");
|
||||
}
|
||||
else
|
||||
{
|
||||
Logger.LogDebug(_TAG, $"Reading the config file ({ConfigFilePath})...");
|
||||
foreach (var line in File.ReadLines(ConfigFilePath))
|
||||
{
|
||||
try
|
||||
{
|
||||
var pair = ParseLine(line);
|
||||
_Config[pair.Item1] = pair.Item2;
|
||||
}
|
||||
catch (System.Exception e)
|
||||
{
|
||||
Logger.LogWarning($"{e}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return _Config;
|
||||
}
|
||||
}
|
||||
|
||||
public static void Commit()
|
||||
{
|
||||
string[] lines = {
|
||||
$"{_GlogLogtostderrKey}={(GlogLogtostderr ? "1" : "0")}",
|
||||
$"{_GlogStderrthresholdKey}={GlogStderrthreshold}",
|
||||
$"{_GlogMinloglevelKey}={GlogMinloglevel}",
|
||||
$"{_GlogLogDirKey}={GlogLogDir}",
|
||||
$"{_GlogVKey}={GlogV}",
|
||||
};
|
||||
if (!Directory.Exists(CacheDirPath))
|
||||
{
|
||||
var _ = Directory.CreateDirectory(CacheDirPath);
|
||||
}
|
||||
File.WriteAllLines(ConfigFilePath, lines, Encoding.UTF8);
|
||||
Logger.LogInfo(_TAG, "Global config file has been updated");
|
||||
}
|
||||
|
||||
public static void SetFlags()
|
||||
{
|
||||
Glog.Logtostderr = GlogLogtostderr;
|
||||
Glog.Stderrthreshold = GlogStderrthreshold;
|
||||
Glog.Minloglevel = GlogMinloglevel;
|
||||
Glog.V = GlogV;
|
||||
Glog.LogDir = GlogLogDir == "" ? null : Path.Combine(Application.persistentDataPath, GlogLogDir);
|
||||
}
|
||||
|
||||
private static (string, string) ParseLine(string line)
|
||||
{
|
||||
var i = line.IndexOf('=');
|
||||
|
||||
if (i < 0)
|
||||
{
|
||||
throw new System.FormatException("Each line in global config file must include '=', but not found");
|
||||
}
|
||||
|
||||
var key = line.Substring(0, i);
|
||||
var value = line.Substring(i + 1);
|
||||
|
||||
return (key, value);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
fileFormatVersion: 2
|
||||
guid: 9b93a8796afa16fd68ef2325d00d4d9a
|
||||
MonoImporter:
|
||||
externalObjects: {}
|
||||
serializedVersion: 2
|
||||
defaultReferences: []
|
||||
executionOrder: 0
|
||||
icon: {instanceID: 0}
|
||||
userData:
|
||||
assetBundleName:
|
||||
assetBundleVariant:
|
||||
311
Assets/MediaPipeUnity/Common/Scripts/GraphRunner.cs
Normal file
311
Assets/MediaPipeUnity/Common/Scripts/GraphRunner.cs
Normal file
@@ -0,0 +1,311 @@
|
||||
// Copyright (c) 2021 homuler
|
||||
//
|
||||
// Use of this source code is governed by an MIT-style
|
||||
// license that can be found in the LICENSE file or at
|
||||
// https://opensource.org/licenses/MIT.
|
||||
|
||||
using System;
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using UnityEngine;
|
||||
using UnityEngine.Rendering;
|
||||
|
||||
using Stopwatch = System.Diagnostics.Stopwatch;
|
||||
|
||||
namespace Mediapipe.Unity
|
||||
{
|
||||
public abstract class GraphRunner : MonoBehaviour
|
||||
{
|
||||
public enum ConfigType
|
||||
{
|
||||
None,
|
||||
CPU,
|
||||
GPU,
|
||||
OpenGLES,
|
||||
}
|
||||
|
||||
#pragma warning disable IDE1006
|
||||
// TODO: make it static
|
||||
protected string TAG => GetType().Name;
|
||||
#pragma warning restore IDE1006
|
||||
|
||||
[SerializeField] private TextAsset _cpuConfig = null;
|
||||
[SerializeField] private TextAsset _gpuConfig = null;
|
||||
[SerializeField] private TextAsset _openGlEsConfig = null;
|
||||
[SerializeField] private long _timeoutMicrosec = 0;
|
||||
|
||||
private static readonly GlobalInstanceTable<int, GraphRunner> _InstanceTable = new GlobalInstanceTable<int, GraphRunner>(5);
|
||||
private static readonly Dictionary<IntPtr, int> _NameTable = new Dictionary<IntPtr, int>();
|
||||
|
||||
protected RunningMode runningMode { get; private set; } = RunningMode.Async;
|
||||
private bool _isRunning = false;
|
||||
|
||||
public InferenceMode inferenceMode => configType == ConfigType.CPU ? InferenceMode.CPU : InferenceMode.GPU;
|
||||
public virtual ConfigType configType
|
||||
{
|
||||
get
|
||||
{
|
||||
if (GpuManager.IsInitialized)
|
||||
{
|
||||
#if UNITY_ANDROID
|
||||
if (SystemInfo.graphicsDeviceType == GraphicsDeviceType.OpenGLES3 && _openGlEsConfig != null)
|
||||
{
|
||||
return ConfigType.OpenGLES;
|
||||
}
|
||||
#endif
|
||||
if (_gpuConfig != null)
|
||||
{
|
||||
return ConfigType.GPU;
|
||||
}
|
||||
}
|
||||
return _cpuConfig != null ? ConfigType.CPU : ConfigType.None;
|
||||
}
|
||||
}
|
||||
public TextAsset textConfig
|
||||
{
|
||||
get
|
||||
{
|
||||
switch (configType)
|
||||
{
|
||||
case ConfigType.CPU: return _cpuConfig;
|
||||
case ConfigType.GPU: return _gpuConfig;
|
||||
case ConfigType.OpenGLES: return _openGlEsConfig;
|
||||
case ConfigType.None:
|
||||
default: return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public long timeoutMicrosec
|
||||
{
|
||||
get => _timeoutMicrosec;
|
||||
set => _timeoutMicrosec = (long)Mathf.Max(0, value);
|
||||
}
|
||||
public long timeoutMillisec
|
||||
{
|
||||
get => timeoutMicrosec / 1000;
|
||||
set => timeoutMicrosec = value * 1000;
|
||||
}
|
||||
|
||||
public RotationAngle rotation { get; private set; } = 0;
|
||||
|
||||
private Stopwatch _stopwatch;
|
||||
protected CalculatorGraph calculatorGraph { get; private set; }
|
||||
protected Timestamp latestTimestamp;
|
||||
|
||||
protected virtual void Start()
|
||||
{
|
||||
_InstanceTable.Add(GetInstanceID(), this);
|
||||
}
|
||||
|
||||
protected virtual void OnDestroy()
|
||||
{
|
||||
Stop();
|
||||
}
|
||||
|
||||
public WaitForResult WaitForInit(RunningMode runningMode)
|
||||
{
|
||||
return new WaitForResult(this, Initialize(runningMode));
|
||||
}
|
||||
|
||||
public virtual IEnumerator Initialize(RunningMode runningMode)
|
||||
{
|
||||
this.runningMode = runningMode;
|
||||
|
||||
Logger.LogInfo(TAG, $"Config Type = {configType}");
|
||||
Logger.LogInfo(TAG, $"Running Mode = {runningMode}");
|
||||
|
||||
InitializeCalculatorGraph().AssertOk();
|
||||
_stopwatch = new Stopwatch();
|
||||
_stopwatch.Start();
|
||||
|
||||
Logger.LogInfo(TAG, "Loading dependent assets...");
|
||||
var assetRequests = RequestDependentAssets();
|
||||
yield return new WaitWhile(() => assetRequests.Any((request) => request.keepWaiting));
|
||||
|
||||
var errors = assetRequests.Where((request) => request.isError).Select((request) => request.error).ToList();
|
||||
if (errors.Count > 0)
|
||||
{
|
||||
foreach (var error in errors)
|
||||
{
|
||||
Logger.LogError(TAG, error);
|
||||
}
|
||||
throw new InternalException("Failed to prepare dependent assets");
|
||||
}
|
||||
}
|
||||
|
||||
public abstract void StartRun(ImageSource imageSource);
|
||||
|
||||
protected void StartRun(SidePacket sidePacket)
|
||||
{
|
||||
calculatorGraph.StartRun(sidePacket).AssertOk();
|
||||
_isRunning = true;
|
||||
}
|
||||
|
||||
public virtual void Stop()
|
||||
{
|
||||
if (calculatorGraph != null)
|
||||
{
|
||||
if (_isRunning)
|
||||
{
|
||||
using (var status = calculatorGraph.CloseAllPacketSources())
|
||||
{
|
||||
if (!status.Ok())
|
||||
{
|
||||
Logger.LogError(TAG, status.ToString());
|
||||
}
|
||||
}
|
||||
|
||||
using (var status = calculatorGraph.WaitUntilDone())
|
||||
{
|
||||
if (!status.Ok())
|
||||
{
|
||||
Logger.LogError(TAG, status.ToString());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
_isRunning = false;
|
||||
var _ = _NameTable.Remove(calculatorGraph.mpPtr);
|
||||
calculatorGraph.Dispose();
|
||||
calculatorGraph = null;
|
||||
}
|
||||
|
||||
if (_stopwatch != null && _stopwatch.IsRunning)
|
||||
{
|
||||
_stopwatch.Stop();
|
||||
}
|
||||
}
|
||||
|
||||
protected void AddPacketToInputStream<T>(string streamName, Packet<T> packet)
|
||||
{
|
||||
calculatorGraph.AddPacketToInputStream(streamName, packet).AssertOk();
|
||||
}
|
||||
|
||||
protected void AddTextureFrameToInputStream(string streamName, TextureFrame textureFrame)
|
||||
{
|
||||
latestTimestamp = GetCurrentTimestamp();
|
||||
|
||||
if (configType == ConfigType.OpenGLES)
|
||||
{
|
||||
var gpuBuffer = textureFrame.BuildGpuBuffer(GpuManager.GlCalculatorHelper.GetGlContext());
|
||||
AddPacketToInputStream(streamName, new GpuBufferPacket(gpuBuffer, latestTimestamp));
|
||||
return;
|
||||
}
|
||||
|
||||
var imageFrame = textureFrame.BuildImageFrame();
|
||||
textureFrame.Release();
|
||||
|
||||
AddPacketToInputStream(streamName, new ImageFramePacket(imageFrame, latestTimestamp));
|
||||
}
|
||||
|
||||
protected bool TryGetNext<TPacket, TValue>(OutputStream<TPacket, TValue> stream, out TValue value, bool allowBlock, long currentTimestampMicrosec) where TPacket : Packet<TValue>, new()
|
||||
{
|
||||
var result = stream.TryGetNext(out value, allowBlock);
|
||||
return result || allowBlock || stream.ResetTimestampIfTimedOut(currentTimestampMicrosec, timeoutMicrosec);
|
||||
}
|
||||
|
||||
protected long GetCurrentTimestampMicrosec()
|
||||
{
|
||||
return _stopwatch == null || !_stopwatch.IsRunning ? -1 : _stopwatch.ElapsedTicks / (TimeSpan.TicksPerMillisecond / 1000);
|
||||
}
|
||||
|
||||
protected Timestamp GetCurrentTimestamp()
|
||||
{
|
||||
var microsec = GetCurrentTimestampMicrosec();
|
||||
return microsec < 0 ? Timestamp.Unset() : new Timestamp(microsec);
|
||||
}
|
||||
|
||||
protected Status InitializeCalculatorGraph()
|
||||
{
|
||||
calculatorGraph = new CalculatorGraph();
|
||||
_NameTable.Add(calculatorGraph.mpPtr, GetInstanceID());
|
||||
|
||||
// NOTE: There's a simpler way to initialize CalculatorGraph.
|
||||
//
|
||||
// calculatorGraph = new CalculatorGraph(config.text);
|
||||
//
|
||||
// However, if the config format is invalid, this code does not initialize CalculatorGraph and does not throw exceptions either.
|
||||
// The problem is that if you call ObserveStreamOutput in this state, the program will crash.
|
||||
// The following code is not very efficient, but it will return Non-OK status when an invalid configuration is given.
|
||||
try
|
||||
{
|
||||
var baseConfig = textConfig == null ? null : CalculatorGraphConfig.Parser.ParseFromTextFormat(textConfig.text);
|
||||
if (baseConfig == null)
|
||||
{
|
||||
throw new InvalidOperationException("Failed to get the text config. Check if the config is set to GraphRunner");
|
||||
}
|
||||
var status = ConfigureCalculatorGraph(baseConfig);
|
||||
return !status.Ok() || inferenceMode == InferenceMode.CPU ? status : calculatorGraph.SetGpuResources(GpuManager.GpuResources);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
return Status.FailedPrecondition(e.ToString());
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Configure and initialize the <see cref="CalculatorGraph" />.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// This is the main process in <see cref="InitializeCalculatorGraph" />.<br />
|
||||
/// At least, <c>calculatorGraph.Initialize</c> must be called here.
|
||||
/// In addition to that, <see cref="OutputStream" /> instances should be initialized.
|
||||
/// </remarks>
|
||||
/// <param name="config">
|
||||
/// A <see cref="CalculatorGraphConfig" /> instance corresponding to <see cref="textConfig" />.<br />
|
||||
/// It can be dynamically modified here.
|
||||
/// </param>
|
||||
protected virtual Status ConfigureCalculatorGraph(CalculatorGraphConfig config)
|
||||
{
|
||||
return calculatorGraph.Initialize(config);
|
||||
}
|
||||
|
||||
protected void SetImageTransformationOptions(SidePacket sidePacket, ImageSource imageSource, bool expectedToBeMirrored = false)
|
||||
{
|
||||
// NOTE: The origin is left-bottom corner in Unity, and right-top corner in MediaPipe.
|
||||
rotation = imageSource.rotation.Reverse();
|
||||
var inputRotation = rotation;
|
||||
var isInverted = CoordinateSystem.ImageCoordinate.IsInverted(rotation);
|
||||
var shouldBeMirrored = imageSource.isHorizontallyFlipped ^ expectedToBeMirrored;
|
||||
var inputHorizontallyFlipped = isInverted ^ shouldBeMirrored;
|
||||
var inputVerticallyFlipped = !isInverted;
|
||||
|
||||
if ((inputHorizontallyFlipped && inputVerticallyFlipped) || rotation == RotationAngle.Rotation180)
|
||||
{
|
||||
inputRotation = inputRotation.Add(RotationAngle.Rotation180);
|
||||
inputHorizontallyFlipped = !inputHorizontallyFlipped;
|
||||
inputVerticallyFlipped = !inputVerticallyFlipped;
|
||||
}
|
||||
|
||||
Logger.LogDebug($"input_rotation = {inputRotation}, input_horizontally_flipped = {inputHorizontallyFlipped}, input_vertically_flipped = {inputVerticallyFlipped}");
|
||||
|
||||
sidePacket.Emplace("input_rotation", new IntPacket((int)inputRotation));
|
||||
sidePacket.Emplace("input_horizontally_flipped", new BoolPacket(inputHorizontallyFlipped));
|
||||
sidePacket.Emplace("input_vertically_flipped", new BoolPacket(inputVerticallyFlipped));
|
||||
}
|
||||
|
||||
protected WaitForResult WaitForAsset(string assetName, string uniqueKey, long timeoutMillisec, bool overwrite = false)
|
||||
{
|
||||
return new WaitForResult(this, AssetLoader.PrepareAssetAsync(assetName, uniqueKey, overwrite), timeoutMillisec);
|
||||
}
|
||||
|
||||
protected WaitForResult WaitForAsset(string assetName, long timeoutMillisec, bool overwrite = false)
|
||||
{
|
||||
return WaitForAsset(assetName, assetName, timeoutMillisec, overwrite);
|
||||
}
|
||||
|
||||
protected WaitForResult WaitForAsset(string assetName, string uniqueKey, bool overwrite = false)
|
||||
{
|
||||
return new WaitForResult(this, AssetLoader.PrepareAssetAsync(assetName, uniqueKey, overwrite));
|
||||
}
|
||||
|
||||
protected WaitForResult WaitForAsset(string assetName, bool overwrite = false)
|
||||
{
|
||||
return WaitForAsset(assetName, assetName, overwrite);
|
||||
}
|
||||
|
||||
protected abstract IList<WaitForResult> RequestDependentAssets();
|
||||
}
|
||||
}
|
||||
11
Assets/MediaPipeUnity/Common/Scripts/GraphRunner.cs.meta
Normal file
11
Assets/MediaPipeUnity/Common/Scripts/GraphRunner.cs.meta
Normal file
@@ -0,0 +1,11 @@
|
||||
fileFormatVersion: 2
|
||||
guid: 00f8c92dc785a7523b94c869aaead45f
|
||||
MonoImporter:
|
||||
externalObjects: {}
|
||||
serializedVersion: 2
|
||||
defaultReferences: []
|
||||
executionOrder: 0
|
||||
icon: {instanceID: 0}
|
||||
userData:
|
||||
assetBundleName:
|
||||
assetBundleVariant:
|
||||
8
Assets/MediaPipeUnity/Common/Scripts/ImageSource.meta
Normal file
8
Assets/MediaPipeUnity/Common/Scripts/ImageSource.meta
Normal file
@@ -0,0 +1,8 @@
|
||||
fileFormatVersion: 2
|
||||
guid: ab516f3bbf28a23489219f7a966fa0cc
|
||||
folderAsset: yes
|
||||
DefaultImporter:
|
||||
externalObjects: {}
|
||||
userData:
|
||||
assetBundleName:
|
||||
assetBundleVariant:
|
||||
159
Assets/MediaPipeUnity/Common/Scripts/ImageSource/ImageSource.cs
Normal file
159
Assets/MediaPipeUnity/Common/Scripts/ImageSource/ImageSource.cs
Normal file
@@ -0,0 +1,159 @@
|
||||
// Copyright (c) 2021 homuler
|
||||
//
|
||||
// Use of this source code is governed by an MIT-style
|
||||
// license that can be found in the LICENSE file or at
|
||||
// https://opensource.org/licenses/MIT.
|
||||
|
||||
using System;
|
||||
using System.Collections;
|
||||
using UnityEngine;
|
||||
using UnityEngine.Experimental.Rendering;
|
||||
|
||||
namespace Mediapipe.Unity
|
||||
{
|
||||
public abstract class ImageSource : MonoBehaviour
|
||||
{
|
||||
[Serializable]
|
||||
public struct ResolutionStruct
|
||||
{
|
||||
public int width;
|
||||
public int height;
|
||||
public double frameRate;
|
||||
|
||||
public ResolutionStruct(int width, int height, double frameRate)
|
||||
{
|
||||
this.width = width;
|
||||
this.height = height;
|
||||
this.frameRate = frameRate;
|
||||
}
|
||||
|
||||
public ResolutionStruct(Resolution resolution)
|
||||
{
|
||||
width = resolution.width;
|
||||
height = resolution.height;
|
||||
frameRate = resolution.refreshRate;
|
||||
}
|
||||
|
||||
public Resolution ToResolution()
|
||||
{
|
||||
return new Resolution() { width = width, height = height, refreshRate = (int)frameRate };
|
||||
}
|
||||
|
||||
public override string ToString()
|
||||
{
|
||||
var aspectRatio = $"{width}x{height}";
|
||||
var frameRateStr = frameRate.ToString("#.##");
|
||||
return frameRate > 0 ? $"{aspectRatio} ({frameRateStr}Hz)" : aspectRatio;
|
||||
}
|
||||
}
|
||||
|
||||
public ResolutionStruct resolution { get; protected set; }
|
||||
|
||||
/// <remarks>
|
||||
/// To call this method, the image source must be prepared.
|
||||
/// </remarks>
|
||||
/// <returns>
|
||||
/// <see cref="TextureFormat" /> that is compatible with the current texture.
|
||||
/// </returns>
|
||||
public TextureFormat textureFormat => isPrepared ? TextureFormatFor(GetCurrentTexture()) : throw new InvalidOperationException("ImageSource is not prepared");
|
||||
public virtual int textureWidth => resolution.width;
|
||||
public virtual int textureHeight => resolution.height;
|
||||
/// <remarks>
|
||||
/// If <see cref="type" /> does not support frame rate, it returns zero.
|
||||
/// </remarks>
|
||||
public virtual double frameRate => resolution.frameRate;
|
||||
public float focalLengthPx { get; } = 2.0f; // TODO: calculate at runtime
|
||||
public virtual bool isHorizontallyFlipped { get; set; } = false;
|
||||
public virtual bool isVerticallyFlipped { get; } = false;
|
||||
public virtual bool isFrontFacing { get; } = false;
|
||||
public virtual RotationAngle rotation { get; } = RotationAngle.Rotation0;
|
||||
|
||||
public abstract string sourceName { get; }
|
||||
public abstract string[] sourceCandidateNames { get; }
|
||||
public abstract ResolutionStruct[] availableResolutions { get; }
|
||||
|
||||
/// <remarks>
|
||||
/// Once <see cref="Play" /> finishes successfully, it will become true.
|
||||
/// </remarks>
|
||||
/// <returns>
|
||||
/// Returns if the image source is prepared.
|
||||
/// </returns>
|
||||
public abstract bool isPrepared { get; }
|
||||
|
||||
public abstract bool isPlaying { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Choose the source from <see cref="sourceCandidateNames" />.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// You need to call <see cref="Play" /> for the change to take effect.
|
||||
/// </remarks>
|
||||
/// <param name="sourceId">The index of <see cref="sourceCandidateNames" /></param>
|
||||
public abstract void SelectSource(int sourceId);
|
||||
|
||||
/// <summary>
|
||||
/// Choose the resolution from <see cref="availableResolutions" />.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// You need to call <see cref="Play" /> for the change to take effect.
|
||||
/// </remarks>
|
||||
/// <param name="resolutionId">The index of <see cref="availableResolutions" /></param>
|
||||
public void SelectResolution(int resolutionId)
|
||||
{
|
||||
var resolutions = availableResolutions;
|
||||
|
||||
if (resolutionId < 0 || resolutionId >= resolutions.Length)
|
||||
{
|
||||
throw new ArgumentException($"Invalid resolution ID: {resolutionId}");
|
||||
}
|
||||
|
||||
resolution = resolutions[resolutionId];
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Prepare image source.
|
||||
/// If <see cref="isPlaying" /> is true, it will reset the image source.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// When it finishes successfully, <see cref="isPrepared" /> will return true.
|
||||
/// </remarks>
|
||||
/// <exception cref="InvalidOperation" />
|
||||
public abstract IEnumerator Play();
|
||||
|
||||
/// <summary>
|
||||
/// Resume image source.
|
||||
/// If <see cref="isPlaying" /> is true, it will do nothing.
|
||||
/// </summary>
|
||||
/// <exception cref="InvalidOperation">
|
||||
/// The image source has not been played.
|
||||
/// </exception>
|
||||
public abstract IEnumerator Resume();
|
||||
|
||||
/// <summary>
|
||||
/// Pause image source.
|
||||
/// If <see cref="isPlaying" /> is false, it will do nothing.
|
||||
/// </summary>
|
||||
public abstract void Pause();
|
||||
|
||||
/// <summary>
|
||||
/// Stop image source.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// When it finishes successfully, <see cref="isPrepared" /> will return false.
|
||||
/// </remarks>
|
||||
public abstract void Stop();
|
||||
|
||||
/// <remarks>
|
||||
/// To call this method, the image source must be prepared.
|
||||
/// </remarks>
|
||||
/// <returns>
|
||||
/// <see cref="Texture" /> that contains the current image.
|
||||
/// </returns>
|
||||
public abstract Texture GetCurrentTexture();
|
||||
|
||||
protected static TextureFormat TextureFormatFor(Texture texture)
|
||||
{
|
||||
return GraphicsFormatUtility.GetTextureFormat(texture.graphicsFormat);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
fileFormatVersion: 2
|
||||
guid: 78116299de071af7094419302302ec05
|
||||
MonoImporter:
|
||||
externalObjects: {}
|
||||
serializedVersion: 2
|
||||
defaultReferences: []
|
||||
executionOrder: 0
|
||||
icon: {instanceID: 0}
|
||||
userData:
|
||||
assetBundleName:
|
||||
assetBundleVariant:
|
||||
@@ -0,0 +1,13 @@
|
||||
// Copyright (c) 2021 homuler
|
||||
//
|
||||
// Use of this source code is governed by an MIT-style
|
||||
// license that can be found in the LICENSE file or at
|
||||
// https://opensource.org/licenses/MIT.
|
||||
|
||||
public enum ImageSourceType
|
||||
{
|
||||
WebCamera = 0,
|
||||
Image = 1,
|
||||
Video = 2,
|
||||
Unknown = 3,
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
fileFormatVersion: 2
|
||||
guid: 2e20c591a3836aae1abc65429b7adde1
|
||||
MonoImporter:
|
||||
externalObjects: {}
|
||||
serializedVersion: 2
|
||||
defaultReferences: []
|
||||
executionOrder: 0
|
||||
icon: {instanceID: 0}
|
||||
userData:
|
||||
assetBundleName:
|
||||
assetBundleVariant:
|
||||
@@ -0,0 +1,136 @@
|
||||
// Copyright (c) 2021 homuler
|
||||
//
|
||||
// Use of this source code is governed by an MIT-style
|
||||
// license that can be found in the LICENSE file or at
|
||||
// https://opensource.org/licenses/MIT.
|
||||
|
||||
using System;
|
||||
using System.Collections;
|
||||
using System.Linq;
|
||||
using UnityEngine;
|
||||
|
||||
namespace Mediapipe.Unity
|
||||
{
|
||||
public class StaticImageSource : ImageSource
|
||||
{
|
||||
[SerializeField] private Texture[] _availableSources;
|
||||
|
||||
[SerializeField]
|
||||
private ResolutionStruct[] _defaultAvailableResolutions = new ResolutionStruct[] {
|
||||
new ResolutionStruct(512, 512, 0),
|
||||
new ResolutionStruct(640, 480, 0),
|
||||
new ResolutionStruct(1280, 720, 0),
|
||||
};
|
||||
|
||||
private Texture2D _outputTexture;
|
||||
private Texture _image;
|
||||
private Texture image
|
||||
{
|
||||
get
|
||||
{
|
||||
if (_image == null && _availableSources != null && _availableSources.Length > 0)
|
||||
{
|
||||
image = _availableSources[0];
|
||||
}
|
||||
return _image;
|
||||
}
|
||||
set
|
||||
{
|
||||
_image = value;
|
||||
resolution = GetDefaultResolution();
|
||||
}
|
||||
}
|
||||
|
||||
public override double frameRate => 0;
|
||||
|
||||
public override string sourceName => image != null ? image.name : null;
|
||||
|
||||
public override string[] sourceCandidateNames => _availableSources?.Select(source => source.name).ToArray();
|
||||
|
||||
public override ResolutionStruct[] availableResolutions => _defaultAvailableResolutions;
|
||||
|
||||
public override bool isPrepared => _outputTexture != null;
|
||||
|
||||
private bool _isPlaying = false;
|
||||
public override bool isPlaying => _isPlaying;
|
||||
|
||||
public override void SelectSource(int sourceId)
|
||||
{
|
||||
if (sourceId < 0 || sourceId >= _availableSources.Length)
|
||||
{
|
||||
throw new ArgumentException($"Invalid source ID: {sourceId}");
|
||||
}
|
||||
|
||||
image = _availableSources[sourceId];
|
||||
}
|
||||
|
||||
public override IEnumerator Play()
|
||||
{
|
||||
if (image == null)
|
||||
{
|
||||
throw new InvalidOperationException("Image is not selected");
|
||||
}
|
||||
if (isPlaying)
|
||||
{
|
||||
yield break;
|
||||
}
|
||||
|
||||
InitializeOutputTexture(image);
|
||||
_isPlaying = true;
|
||||
yield return null;
|
||||
}
|
||||
|
||||
public override IEnumerator Resume()
|
||||
{
|
||||
if (!isPrepared)
|
||||
{
|
||||
throw new InvalidOperationException("Image is not prepared");
|
||||
}
|
||||
_isPlaying = true;
|
||||
|
||||
yield return null;
|
||||
}
|
||||
|
||||
public override void Pause()
|
||||
{
|
||||
_isPlaying = false;
|
||||
}
|
||||
public override void Stop()
|
||||
{
|
||||
_isPlaying = false;
|
||||
_outputTexture = null;
|
||||
}
|
||||
|
||||
public override Texture GetCurrentTexture()
|
||||
{
|
||||
return _outputTexture;
|
||||
}
|
||||
|
||||
private ResolutionStruct GetDefaultResolution()
|
||||
{
|
||||
var resolutions = availableResolutions;
|
||||
|
||||
return (resolutions == null || resolutions.Length == 0) ? new ResolutionStruct() : resolutions[0];
|
||||
}
|
||||
|
||||
private void InitializeOutputTexture(Texture src)
|
||||
{
|
||||
_outputTexture = new Texture2D(textureWidth, textureHeight, TextureFormat.RGBA32, false);
|
||||
|
||||
Texture resizedTexture = new Texture2D(textureWidth, textureHeight, TextureFormat.RGBA32, false);
|
||||
// TODO: assert ConvertTexture finishes successfully
|
||||
var _ = Graphics.ConvertTexture(src, resizedTexture);
|
||||
|
||||
var currentRenderTexture = RenderTexture.active;
|
||||
var tmpRenderTexture = new RenderTexture(resizedTexture.width, resizedTexture.height, 32);
|
||||
Graphics.Blit(resizedTexture, tmpRenderTexture);
|
||||
RenderTexture.active = tmpRenderTexture;
|
||||
|
||||
var rect = new UnityEngine.Rect(0, 0, _outputTexture.width, _outputTexture.height);
|
||||
_outputTexture.ReadPixels(rect, 0, 0);
|
||||
_outputTexture.Apply();
|
||||
|
||||
RenderTexture.active = currentRenderTexture;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
fileFormatVersion: 2
|
||||
guid: bd7955705ab46c72b9124bb116a2dca9
|
||||
MonoImporter:
|
||||
externalObjects: {}
|
||||
serializedVersion: 2
|
||||
defaultReferences: []
|
||||
executionOrder: 0
|
||||
icon: {instanceID: 0}
|
||||
userData:
|
||||
assetBundleName:
|
||||
assetBundleVariant:
|
||||
378
Assets/MediaPipeUnity/Common/Scripts/ImageSource/TextureFrame.cs
Normal file
378
Assets/MediaPipeUnity/Common/Scripts/ImageSource/TextureFrame.cs
Normal file
@@ -0,0 +1,378 @@
|
||||
// Copyright (c) 2021 homuler
|
||||
//
|
||||
// Use of this source code is governed by an MIT-style
|
||||
// license that can be found in the LICENSE file or at
|
||||
// https://opensource.org/licenses/MIT.
|
||||
|
||||
using System;
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
using Unity.Collections;
|
||||
using UnityEngine;
|
||||
using UnityEngine.Events;
|
||||
using UnityEngine.Experimental.Rendering;
|
||||
|
||||
namespace Mediapipe.Unity
|
||||
{
|
||||
#pragma warning disable IDE0065
|
||||
using Color = UnityEngine.Color;
|
||||
#pragma warning restore IDE0065
|
||||
|
||||
public class TextureFrame
|
||||
{
|
||||
public class ReleaseEvent : UnityEvent<TextureFrame> { }
|
||||
|
||||
private const string _TAG = nameof(TextureFrame);
|
||||
|
||||
private static readonly GlobalInstanceTable<Guid, TextureFrame> _InstanceTable = new GlobalInstanceTable<Guid, TextureFrame>(100);
|
||||
/// <summary>
|
||||
/// A dictionary to look up which native texture belongs to which <see cref="TextureFrame" />.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Not all the <see cref="TextureFrame" /> instances are registered.
|
||||
/// Texture names are queried only when necessary, and the corresponding data will be saved then.
|
||||
/// </remarks>
|
||||
private static readonly Dictionary<uint, Guid> _NameTable = new Dictionary<uint, Guid>();
|
||||
|
||||
private readonly Texture2D _texture;
|
||||
private IntPtr _nativeTexturePtr = IntPtr.Zero;
|
||||
private GlSyncPoint _glSyncToken;
|
||||
|
||||
// Buffers that will be used to copy texture data on CPU.
|
||||
// They won't be initialized until it's necessary.
|
||||
private Texture2D _textureBuffer;
|
||||
|
||||
private Color32[] _pixelsBuffer; // for WebCamTexture
|
||||
private Color32[] pixelsBuffer
|
||||
{
|
||||
get
|
||||
{
|
||||
if (_pixelsBuffer == null)
|
||||
{
|
||||
_pixelsBuffer = new Color32[width * height];
|
||||
}
|
||||
return _pixelsBuffer;
|
||||
}
|
||||
}
|
||||
|
||||
private readonly Guid _instanceId;
|
||||
// NOTE: width and height can be accessed from a thread other than Main Thread.
|
||||
public readonly int width;
|
||||
public readonly int height;
|
||||
public readonly TextureFormat format;
|
||||
|
||||
private ImageFormat.Types.Format _format = ImageFormat.Types.Format.Unknown;
|
||||
public ImageFormat.Types.Format imageFormat
|
||||
{
|
||||
get
|
||||
{
|
||||
if (_format == ImageFormat.Types.Format.Unknown)
|
||||
{
|
||||
_format = format.ToImageFormat();
|
||||
}
|
||||
return _format;
|
||||
}
|
||||
}
|
||||
|
||||
public bool isReadable => _texture.isReadable;
|
||||
|
||||
// TODO: determine at runtime
|
||||
public GpuBufferFormat gpuBufferformat => GpuBufferFormat.kBGRA32;
|
||||
|
||||
/// <summary>
|
||||
/// The event that will be invoked when the TextureFrame is released.
|
||||
/// </summary>
|
||||
#pragma warning disable IDE1006 // UnityEvent is PascalCase
|
||||
public readonly ReleaseEvent OnRelease;
|
||||
#pragma warning restore IDE1006
|
||||
|
||||
private TextureFrame(Texture2D texture)
|
||||
{
|
||||
_texture = texture;
|
||||
width = texture.width;
|
||||
height = texture.height;
|
||||
format = texture.format;
|
||||
OnRelease = new ReleaseEvent();
|
||||
_instanceId = Guid.NewGuid();
|
||||
_InstanceTable.Add(_instanceId, this);
|
||||
}
|
||||
|
||||
public TextureFrame(int width, int height, TextureFormat format) : this(new Texture2D(width, height, format, false)) { }
|
||||
public TextureFrame(int width, int height) : this(width, height, TextureFormat.RGBA32) { }
|
||||
|
||||
public void CopyTexture(Texture dst)
|
||||
{
|
||||
Graphics.CopyTexture(_texture, dst);
|
||||
}
|
||||
|
||||
public void CopyTextureFrom(Texture src)
|
||||
{
|
||||
Graphics.CopyTexture(src, _texture);
|
||||
}
|
||||
|
||||
public bool ConvertTexture(Texture dst)
|
||||
{
|
||||
return Graphics.ConvertTexture(_texture, dst);
|
||||
}
|
||||
|
||||
public bool ConvertTextureFrom(Texture src)
|
||||
{
|
||||
return Graphics.ConvertTexture(src, _texture);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Copy texture data from <paramref name="src" />.
|
||||
/// If <paramref name="src" /> format is different from <see cref="format" />, it converts the format.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// After calling it, pixel data won't be read from CPU safely.
|
||||
/// </remarks>
|
||||
public bool ReadTextureFromOnGPU(Texture src)
|
||||
{
|
||||
if (GetTextureFormat(src) != format)
|
||||
{
|
||||
return Graphics.ConvertTexture(src, _texture);
|
||||
}
|
||||
Graphics.CopyTexture(src, _texture);
|
||||
return true;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Copy texture data from <paramref name="src" />.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// This operation is slow.
|
||||
/// If CPU won't access the pixel data, use <see cref="ReadTextureFromOnGPU" /> instead.
|
||||
/// </remarks>
|
||||
public void ReadTextureFromOnCPU(Texture src)
|
||||
{
|
||||
var textureBuffer = LoadToTextureBuffer(src);
|
||||
SetPixels32(textureBuffer.GetPixels32());
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Copy texture data from <paramref name="src" />.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// In most cases, it should be better to use <paramref name="src" /> directly.
|
||||
/// </remarks>
|
||||
public void ReadTextureFromOnCPU(Texture2D src)
|
||||
{
|
||||
SetPixels32(src.GetPixels32());
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Copy texture data from <paramref name="src" />.
|
||||
/// </summary>
|
||||
/// <param name="src">
|
||||
/// The texture from which the pixels are read.
|
||||
/// Its width and height must match that of the TextureFrame.
|
||||
/// </param>
|
||||
/// <remarks>
|
||||
/// This operation is slow.
|
||||
/// If CPU won't access the pixel data, use <see cref="ReadTextureFromOnGPU" /> instead.
|
||||
/// </remarks>
|
||||
public void ReadTextureFromOnCPU(WebCamTexture src)
|
||||
{
|
||||
SetPixels32(src.GetPixels32(pixelsBuffer));
|
||||
}
|
||||
|
||||
public Color GetPixel(int x, int y)
|
||||
{
|
||||
return _texture.GetPixel(x, y);
|
||||
}
|
||||
|
||||
public Color32[] GetPixels32()
|
||||
{
|
||||
return _texture.GetPixels32();
|
||||
}
|
||||
|
||||
public void SetPixels32(Color32[] pixels)
|
||||
{
|
||||
_texture.SetPixels32(pixels);
|
||||
_texture.Apply();
|
||||
|
||||
if (!RevokeNativeTexturePtr())
|
||||
{
|
||||
// If this line was executed, there must be a bug.
|
||||
Logger.LogError("Failed to revoke the native texture.");
|
||||
}
|
||||
}
|
||||
|
||||
public NativeArray<T> GetRawTextureData<T>() where T : struct
|
||||
{
|
||||
return _texture.GetRawTextureData<T>();
|
||||
}
|
||||
|
||||
/// <returns>The texture's native pointer</returns>
|
||||
public IntPtr GetNativeTexturePtr()
|
||||
{
|
||||
if (_nativeTexturePtr == IntPtr.Zero)
|
||||
{
|
||||
_nativeTexturePtr = _texture.GetNativeTexturePtr();
|
||||
var name = (uint)_nativeTexturePtr;
|
||||
|
||||
lock (((ICollection)_NameTable).SyncRoot)
|
||||
{
|
||||
if (!AcquireName(name, _instanceId))
|
||||
{
|
||||
throw new InvalidProgramException($"Another instance (id={_instanceId}) is using the specified name ({name}) now");
|
||||
}
|
||||
_NameTable.Add(name, _instanceId);
|
||||
}
|
||||
}
|
||||
return _nativeTexturePtr;
|
||||
}
|
||||
|
||||
public uint GetTextureName()
|
||||
{
|
||||
return (uint)GetNativeTexturePtr();
|
||||
}
|
||||
|
||||
public Guid GetInstanceID()
|
||||
{
|
||||
return _instanceId;
|
||||
}
|
||||
|
||||
public ImageFrame BuildImageFrame()
|
||||
{
|
||||
return new ImageFrame(imageFormat, width, height, 4 * width, GetRawTextureData<byte>());
|
||||
}
|
||||
|
||||
public GpuBuffer BuildGpuBuffer(GlContext glContext)
|
||||
{
|
||||
#if UNITY_EDITOR_LINUX || UNITY_STANDALONE_LINUX || UNITY_ANDROID
|
||||
var glTextureBuffer = new GlTextureBuffer(GetTextureName(), width, height, gpuBufferformat, OnReleaseTextureFrame, glContext);
|
||||
return new GpuBuffer(glTextureBuffer);
|
||||
#else
|
||||
throw new NotSupportedException("This method is only supported on Linux or Android");
|
||||
#endif
|
||||
}
|
||||
|
||||
public void RemoveAllReleaseListeners()
|
||||
{
|
||||
OnRelease.RemoveAllListeners();
|
||||
}
|
||||
|
||||
// TODO: stop invoking OnRelease when it's already released
|
||||
public void Release(GlSyncPoint token = null)
|
||||
{
|
||||
if (_glSyncToken != null)
|
||||
{
|
||||
_glSyncToken.Dispose();
|
||||
}
|
||||
_glSyncToken = token;
|
||||
OnRelease.Invoke(this);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Waits until the GPU has executed all commands up to the sync point.
|
||||
/// This blocks the CPU, and ensures the commands are complete from the point of view of all threads and contexts.
|
||||
/// </summary>
|
||||
public void WaitUntilReleased()
|
||||
{
|
||||
if (_glSyncToken == null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
_glSyncToken.Wait();
|
||||
_glSyncToken.Dispose();
|
||||
_glSyncToken = null;
|
||||
}
|
||||
|
||||
[AOT.MonoPInvokeCallback(typeof(GlTextureBuffer.DeletionCallback))]
|
||||
public static void OnReleaseTextureFrame(uint textureName, IntPtr syncTokenPtr)
|
||||
{
|
||||
var isIdFound = _NameTable.TryGetValue(textureName, out var _instanceId);
|
||||
|
||||
if (!isIdFound)
|
||||
{
|
||||
Logger.LogError(_TAG, $"nameof (name={textureName}) is released, but the owner TextureFrame is not found");
|
||||
return;
|
||||
}
|
||||
|
||||
var isTextureFrameFound = _InstanceTable.TryGetValue(_instanceId, out var textureFrame);
|
||||
|
||||
if (!isTextureFrameFound)
|
||||
{
|
||||
Logger.LogWarning(_TAG, $"nameof owner TextureFrame of the released texture (name={textureName}) is already garbage collected");
|
||||
return;
|
||||
}
|
||||
|
||||
var _glSyncToken = syncTokenPtr == IntPtr.Zero ? null : new GlSyncPoint(syncTokenPtr);
|
||||
textureFrame.Release(_glSyncToken);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Remove <paramref name="name" /> from <see cref="_NameTable" /> if it's stale.
|
||||
/// If <paramref name="name" /> does not exist in <see cref="_NameTable" />, do nothing.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// If the instance whose id is <paramref name="ownerId" /> owns <paramref name="name" /> now, it still removes <paramref name="name" />.
|
||||
/// </remarks>
|
||||
/// <returns>Return if name is available</returns>
|
||||
private static bool AcquireName(uint name, Guid ownerId)
|
||||
{
|
||||
if (_NameTable.TryGetValue(name, out var id))
|
||||
{
|
||||
if (ownerId != id && _InstanceTable.TryGetValue(id, out var _))
|
||||
{
|
||||
// if instance is found, the instance is using the name.
|
||||
Logger.LogVerbose($"{id} is using {name} now");
|
||||
return false;
|
||||
}
|
||||
var _ = _NameTable.Remove(name);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
private static TextureFormat GetTextureFormat(Texture texture)
|
||||
{
|
||||
return GraphicsFormatUtility.GetTextureFormat(texture.graphicsFormat);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Remove the texture name from <see cref="_NameTable" /> and empty <see cref="_nativeTexturePtr" />.
|
||||
/// This method needs to be called when an operation is performed that may change the internal texture.
|
||||
/// </summary>
|
||||
private bool RevokeNativeTexturePtr()
|
||||
{
|
||||
if (_nativeTexturePtr == IntPtr.Zero)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
var currentName = GetTextureName();
|
||||
if (!_NameTable.Remove(currentName))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
_nativeTexturePtr = IntPtr.Zero;
|
||||
return true;
|
||||
}
|
||||
|
||||
private Texture2D LoadToTextureBuffer(Texture texture)
|
||||
{
|
||||
var textureFormat = GetTextureFormat(texture);
|
||||
|
||||
if (_textureBuffer == null || _textureBuffer.format != textureFormat)
|
||||
{
|
||||
_textureBuffer = new Texture2D(width, height, textureFormat, false);
|
||||
}
|
||||
|
||||
var tmpRenderTexture = RenderTexture.GetTemporary(texture.width, texture.height, 32);
|
||||
var currentRenderTexture = RenderTexture.active;
|
||||
RenderTexture.active = tmpRenderTexture;
|
||||
|
||||
Graphics.Blit(texture, tmpRenderTexture);
|
||||
|
||||
var rect = new UnityEngine.Rect(0, 0, Mathf.Min(tmpRenderTexture.width, _textureBuffer.width), Mathf.Min(tmpRenderTexture.height, _textureBuffer.height));
|
||||
_textureBuffer.ReadPixels(rect, 0, 0);
|
||||
_textureBuffer.Apply();
|
||||
RenderTexture.active = currentRenderTexture;
|
||||
RenderTexture.ReleaseTemporary(tmpRenderTexture);
|
||||
|
||||
return _textureBuffer;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
fileFormatVersion: 2
|
||||
guid: fa7e0da68d497cd578438e238b6a6e7c
|
||||
MonoImporter:
|
||||
externalObjects: {}
|
||||
serializedVersion: 2
|
||||
defaultReferences: []
|
||||
executionOrder: 0
|
||||
icon: {instanceID: 0}
|
||||
userData:
|
||||
assetBundleName:
|
||||
assetBundleVariant:
|
||||
@@ -0,0 +1,161 @@
|
||||
// Copyright (c) 2021 homuler
|
||||
//
|
||||
// Use of this source code is governed by an MIT-style
|
||||
// license that can be found in the LICENSE file or at
|
||||
// https://opensource.org/licenses/MIT.
|
||||
|
||||
using System;
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
using UnityEngine;
|
||||
|
||||
namespace Mediapipe.Unity
|
||||
{
|
||||
public class TextureFramePool : MonoBehaviour
|
||||
{
|
||||
private const string _TAG = nameof(TextureFramePool);
|
||||
|
||||
[SerializeField] private int _poolSize = 10;
|
||||
|
||||
private readonly object _formatLock = new object();
|
||||
private int _textureWidth = 0;
|
||||
private int _textureHeight = 0;
|
||||
private TextureFormat _format = TextureFormat.RGBA32;
|
||||
|
||||
private Queue<TextureFrame> _availableTextureFrames;
|
||||
/// <remarks>
|
||||
/// key: TextureFrame's instance ID
|
||||
/// </remarks>
|
||||
private Dictionary<Guid, TextureFrame> _textureFramesInUse;
|
||||
|
||||
/// <returns>
|
||||
/// The total number of texture frames in the pool.
|
||||
/// </returns>
|
||||
public int frameCount
|
||||
{
|
||||
get
|
||||
{
|
||||
var availableTextureFramesCount = _availableTextureFrames == null ? 0 : _availableTextureFrames.Count;
|
||||
var textureFramesInUseCount = _textureFramesInUse == null ? 0 : _textureFramesInUse.Count;
|
||||
|
||||
return availableTextureFramesCount + textureFramesInUseCount;
|
||||
}
|
||||
}
|
||||
|
||||
private void Start()
|
||||
{
|
||||
_availableTextureFrames = new Queue<TextureFrame>(_poolSize);
|
||||
_textureFramesInUse = new Dictionary<Guid, TextureFrame>();
|
||||
}
|
||||
|
||||
private void OnDestroy()
|
||||
{
|
||||
lock (((ICollection)_availableTextureFrames).SyncRoot)
|
||||
{
|
||||
_availableTextureFrames.Clear();
|
||||
_availableTextureFrames = null;
|
||||
}
|
||||
|
||||
lock (((ICollection)_textureFramesInUse).SyncRoot)
|
||||
{
|
||||
foreach (var textureFrame in _textureFramesInUse.Values)
|
||||
{
|
||||
textureFrame.OnRelease.RemoveListener(OnTextureFrameRelease);
|
||||
}
|
||||
_textureFramesInUse.Clear();
|
||||
_textureFramesInUse = null;
|
||||
}
|
||||
}
|
||||
|
||||
public void ResizeTexture(int textureWidth, int textureHeight, TextureFormat format)
|
||||
{
|
||||
lock (_formatLock)
|
||||
{
|
||||
_textureWidth = textureWidth;
|
||||
_textureHeight = textureHeight;
|
||||
_format = format;
|
||||
}
|
||||
}
|
||||
|
||||
public void ResizeTexture(int textureWidth, int textureHeight)
|
||||
{
|
||||
ResizeTexture(textureWidth, textureHeight, _format);
|
||||
}
|
||||
|
||||
public bool TryGetTextureFrame(out TextureFrame outFrame)
|
||||
{
|
||||
TextureFrame nextFrame = null;
|
||||
|
||||
lock (((ICollection)_availableTextureFrames).SyncRoot)
|
||||
{
|
||||
if (_poolSize <= frameCount)
|
||||
{
|
||||
while (_availableTextureFrames.Count > 0)
|
||||
{
|
||||
var textureFrame = _availableTextureFrames.Dequeue();
|
||||
|
||||
if (!IsStale(textureFrame))
|
||||
{
|
||||
nextFrame = textureFrame;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
nextFrame = CreateNewTextureFrame();
|
||||
}
|
||||
}
|
||||
|
||||
if (nextFrame == null)
|
||||
{
|
||||
outFrame = null;
|
||||
return false;
|
||||
}
|
||||
|
||||
lock (((ICollection)_textureFramesInUse).SyncRoot)
|
||||
{
|
||||
_textureFramesInUse.Add(nextFrame.GetInstanceID(), nextFrame);
|
||||
}
|
||||
|
||||
nextFrame.WaitUntilReleased();
|
||||
outFrame = nextFrame;
|
||||
return true;
|
||||
}
|
||||
|
||||
private void OnTextureFrameRelease(TextureFrame textureFrame)
|
||||
{
|
||||
lock (((ICollection)_textureFramesInUse).SyncRoot)
|
||||
{
|
||||
if (!_textureFramesInUse.Remove(textureFrame.GetInstanceID()))
|
||||
{
|
||||
// won't be run
|
||||
Logger.LogWarning(_TAG, "The released texture does not belong to the pool");
|
||||
return;
|
||||
}
|
||||
|
||||
if (frameCount > _poolSize || IsStale(textureFrame))
|
||||
{
|
||||
return;
|
||||
}
|
||||
_availableTextureFrames.Enqueue(textureFrame);
|
||||
}
|
||||
}
|
||||
|
||||
private bool IsStale(TextureFrame textureFrame)
|
||||
{
|
||||
lock (_formatLock)
|
||||
{
|
||||
return textureFrame.width != _textureWidth || textureFrame.height != _textureHeight;
|
||||
}
|
||||
}
|
||||
|
||||
private TextureFrame CreateNewTextureFrame()
|
||||
{
|
||||
var textureFrame = new TextureFrame(_textureWidth, _textureHeight, _format);
|
||||
textureFrame.OnRelease.AddListener(OnTextureFrameRelease);
|
||||
|
||||
return textureFrame;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
fileFormatVersion: 2
|
||||
guid: d5da564da19cb6b7d8e4f97f269edc5d
|
||||
MonoImporter:
|
||||
externalObjects: {}
|
||||
serializedVersion: 2
|
||||
defaultReferences: []
|
||||
executionOrder: 0
|
||||
icon: {instanceID: 0}
|
||||
userData:
|
||||
assetBundleName:
|
||||
assetBundleVariant:
|
||||
116
Assets/MediaPipeUnity/Common/Scripts/ImageSource/VideoSource.cs
Normal file
116
Assets/MediaPipeUnity/Common/Scripts/ImageSource/VideoSource.cs
Normal file
@@ -0,0 +1,116 @@
|
||||
// Copyright (c) 2021 homuler
|
||||
//
|
||||
// Use of this source code is governed by an MIT-style
|
||||
// license that can be found in the LICENSE file or at
|
||||
// https://opensource.org/licenses/MIT.
|
||||
|
||||
using System;
|
||||
using System.Collections;
|
||||
using System.Linq;
|
||||
using UnityEngine;
|
||||
using UnityEngine.Video;
|
||||
|
||||
namespace Mediapipe.Unity
|
||||
{
|
||||
public class VideoSource : ImageSource
|
||||
{
|
||||
[SerializeField] private VideoClip[] _availableSources;
|
||||
|
||||
private VideoClip _video;
|
||||
private VideoClip video
|
||||
{
|
||||
get
|
||||
{
|
||||
if (_video == null && _availableSources != null && _availableSources.Length > 0)
|
||||
{
|
||||
video = _availableSources[0];
|
||||
}
|
||||
return _video;
|
||||
}
|
||||
set
|
||||
{
|
||||
_video = value;
|
||||
resolution = new ResolutionStruct((int)_video.width, (int)_video.height, _video.frameRate);
|
||||
}
|
||||
}
|
||||
|
||||
private VideoPlayer _videoPlayer;
|
||||
|
||||
public override string sourceName => video != null ? video.name : null;
|
||||
|
||||
public override string[] sourceCandidateNames => _availableSources?.Select(source => source.name).ToArray();
|
||||
|
||||
public override ResolutionStruct[] availableResolutions => video == null ? null : new ResolutionStruct[] { new ResolutionStruct((int)video.width, (int)video.height, video.frameRate) };
|
||||
|
||||
public override bool isPlaying => _videoPlayer != null && _videoPlayer.isPlaying;
|
||||
public override bool isPrepared => _videoPlayer != null && _videoPlayer.isPrepared;
|
||||
|
||||
public override void SelectSource(int sourceId)
|
||||
{
|
||||
if (sourceId < 0 || sourceId >= _availableSources.Length)
|
||||
{
|
||||
throw new ArgumentException($"Invalid source ID: {sourceId}");
|
||||
}
|
||||
|
||||
video = _availableSources[sourceId];
|
||||
if (_videoPlayer != null)
|
||||
{
|
||||
_videoPlayer.clip = video;
|
||||
}
|
||||
}
|
||||
|
||||
public override IEnumerator Play()
|
||||
{
|
||||
if (video == null)
|
||||
{
|
||||
throw new InvalidOperationException("Video is not selected");
|
||||
}
|
||||
_videoPlayer = gameObject.AddComponent<VideoPlayer>();
|
||||
_videoPlayer.renderMode = VideoRenderMode.APIOnly;
|
||||
_videoPlayer.isLooping = true;
|
||||
_videoPlayer.clip = video;
|
||||
_videoPlayer.Prepare();
|
||||
|
||||
yield return new WaitUntil(() => _videoPlayer.isPrepared);
|
||||
_videoPlayer.Play();
|
||||
}
|
||||
|
||||
public override IEnumerator Resume()
|
||||
{
|
||||
if (!isPrepared)
|
||||
{
|
||||
throw new InvalidOperationException("VideoPlayer is not prepared");
|
||||
}
|
||||
if (!isPlaying)
|
||||
{
|
||||
_videoPlayer.Play();
|
||||
}
|
||||
yield return null;
|
||||
}
|
||||
|
||||
public override void Pause()
|
||||
{
|
||||
if (!isPlaying)
|
||||
{
|
||||
return;
|
||||
}
|
||||
_videoPlayer.Pause();
|
||||
}
|
||||
|
||||
public override void Stop()
|
||||
{
|
||||
if (_videoPlayer == null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
_videoPlayer.Stop();
|
||||
Destroy(gameObject.GetComponent<VideoPlayer>());
|
||||
_videoPlayer = null;
|
||||
}
|
||||
|
||||
public override Texture GetCurrentTexture()
|
||||
{
|
||||
return _videoPlayer != null ? _videoPlayer.texture : null;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
fileFormatVersion: 2
|
||||
guid: 04085488e5fac35599866a2a6fceeda3
|
||||
MonoImporter:
|
||||
externalObjects: {}
|
||||
serializedVersion: 2
|
||||
defaultReferences: []
|
||||
executionOrder: 0
|
||||
icon: {instanceID: 0}
|
||||
userData:
|
||||
assetBundleName:
|
||||
assetBundleVariant:
|
||||
299
Assets/MediaPipeUnity/Common/Scripts/ImageSource/WebCamSource.cs
Normal file
299
Assets/MediaPipeUnity/Common/Scripts/ImageSource/WebCamSource.cs
Normal file
@@ -0,0 +1,299 @@
|
||||
// Copyright (c) 2021 homuler
|
||||
//
|
||||
// Use of this source code is governed by an MIT-style
|
||||
// license that can be found in the LICENSE file or at
|
||||
// https://opensource.org/licenses/MIT.
|
||||
|
||||
using System;
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using UnityEngine;
|
||||
|
||||
#if UNITY_ANDROID
|
||||
using UnityEngine.Android;
|
||||
#endif
|
||||
|
||||
namespace Mediapipe.Unity
|
||||
{
|
||||
public class WebCamSource : ImageSource
|
||||
{
|
||||
[Tooltip("For the default resolution, the one whose width is closest to this value will be chosen")]
|
||||
[SerializeField] private int _preferableDefaultWidth = 1280;
|
||||
|
||||
private const string _TAG = nameof(WebCamSource);
|
||||
|
||||
[SerializeField]
|
||||
private ResolutionStruct[] _defaultAvailableResolutions = new ResolutionStruct[] {
|
||||
new ResolutionStruct(176, 144, 30),
|
||||
new ResolutionStruct(320, 240, 30),
|
||||
new ResolutionStruct(424, 240, 30),
|
||||
new ResolutionStruct(640, 360, 30),
|
||||
new ResolutionStruct(640, 480, 30),
|
||||
new ResolutionStruct(848, 480, 30),
|
||||
new ResolutionStruct(960, 540, 30),
|
||||
new ResolutionStruct(1280, 720, 30),
|
||||
new ResolutionStruct(1600, 896, 30),
|
||||
new ResolutionStruct(1920, 1080, 30),
|
||||
};
|
||||
|
||||
private static readonly object _PermissionLock = new object();
|
||||
private static bool _IsPermitted = false;
|
||||
|
||||
private WebCamTexture _webCamTexture;
|
||||
private WebCamTexture webCamTexture
|
||||
{
|
||||
get => _webCamTexture;
|
||||
set
|
||||
{
|
||||
if (_webCamTexture != null)
|
||||
{
|
||||
_webCamTexture.Stop();
|
||||
}
|
||||
_webCamTexture = value;
|
||||
}
|
||||
}
|
||||
|
||||
public override int textureWidth => !isPrepared ? 0 : webCamTexture.width;
|
||||
public override int textureHeight => !isPrepared ? 0 : webCamTexture.height;
|
||||
|
||||
public override bool isVerticallyFlipped => isPrepared && webCamTexture.videoVerticallyMirrored;
|
||||
public override bool isFrontFacing => isPrepared && (webCamDevice is WebCamDevice valueOfWebCamDevice) && valueOfWebCamDevice.isFrontFacing;
|
||||
public override RotationAngle rotation => !isPrepared ? RotationAngle.Rotation0 : (RotationAngle)webCamTexture.videoRotationAngle;
|
||||
|
||||
private WebCamDevice? _webCamDevice;
|
||||
private WebCamDevice? webCamDevice
|
||||
{
|
||||
get => _webCamDevice;
|
||||
set
|
||||
{
|
||||
if (_webCamDevice is WebCamDevice valueOfWebCamDevice)
|
||||
{
|
||||
if (value is WebCamDevice valueOfValue && valueOfValue.name == valueOfWebCamDevice.name)
|
||||
{
|
||||
// not changed
|
||||
return;
|
||||
}
|
||||
}
|
||||
else if (value == null)
|
||||
{
|
||||
// not changed
|
||||
return;
|
||||
}
|
||||
_webCamDevice = value;
|
||||
resolution = GetDefaultResolution();
|
||||
}
|
||||
}
|
||||
public override string sourceName => (webCamDevice is WebCamDevice valueOfWebCamDevice) ? valueOfWebCamDevice.name : null;
|
||||
|
||||
private WebCamDevice[] _availableSources;
|
||||
private WebCamDevice[] availableSources
|
||||
{
|
||||
get
|
||||
{
|
||||
if (_availableSources == null)
|
||||
{
|
||||
_availableSources = WebCamTexture.devices;
|
||||
}
|
||||
|
||||
return _availableSources;
|
||||
}
|
||||
set => _availableSources = value;
|
||||
}
|
||||
|
||||
public override string[] sourceCandidateNames => availableSources?.Select(device => device.name).ToArray();
|
||||
|
||||
#pragma warning disable IDE0025
|
||||
public override ResolutionStruct[] availableResolutions
|
||||
{
|
||||
get
|
||||
{
|
||||
#if (UNITY_ANDROID || UNITY_IOS) && !UNITY_EDITOR
|
||||
if (webCamDevice is WebCamDevice valueOfWebCamDevice) {
|
||||
return valueOfWebCamDevice.availableResolutions.Select(resolution => new ResolutionStruct(resolution)).ToArray();
|
||||
}
|
||||
#endif
|
||||
return webCamDevice == null ? null : _defaultAvailableResolutions;
|
||||
}
|
||||
}
|
||||
#pragma warning restore IDE0025
|
||||
|
||||
public override bool isPrepared => webCamTexture != null;
|
||||
public override bool isPlaying => webCamTexture != null && webCamTexture.isPlaying;
|
||||
private bool _isInitialized;
|
||||
|
||||
private IEnumerator Start()
|
||||
{
|
||||
yield return GetPermission();
|
||||
|
||||
if (!_IsPermitted)
|
||||
{
|
||||
_isInitialized = true;
|
||||
yield break;
|
||||
}
|
||||
|
||||
availableSources = WebCamTexture.devices;
|
||||
|
||||
if (availableSources != null && availableSources.Length > 0)
|
||||
{
|
||||
webCamDevice = availableSources[0];
|
||||
}
|
||||
|
||||
_isInitialized = true;
|
||||
}
|
||||
|
||||
private IEnumerator GetPermission()
|
||||
{
|
||||
lock (_PermissionLock)
|
||||
{
|
||||
if (_IsPermitted)
|
||||
{
|
||||
yield break;
|
||||
}
|
||||
|
||||
#if UNITY_ANDROID
|
||||
if (!Permission.HasUserAuthorizedPermission(Permission.Camera))
|
||||
{
|
||||
Permission.RequestUserPermission(Permission.Camera);
|
||||
yield return new WaitForSeconds(0.1f);
|
||||
}
|
||||
#elif UNITY_IOS
|
||||
if (!Application.HasUserAuthorization(UserAuthorization.WebCam)) {
|
||||
yield return Application.RequestUserAuthorization(UserAuthorization.WebCam);
|
||||
}
|
||||
#endif
|
||||
|
||||
#if UNITY_ANDROID
|
||||
if (!Permission.HasUserAuthorizedPermission(Permission.Camera))
|
||||
{
|
||||
Logger.LogWarning(_TAG, "Not permitted to use Camera");
|
||||
yield break;
|
||||
}
|
||||
#elif UNITY_IOS
|
||||
if (!Application.HasUserAuthorization(UserAuthorization.WebCam)) {
|
||||
Logger.LogWarning(_TAG, "Not permitted to use WebCam");
|
||||
yield break;
|
||||
}
|
||||
#endif
|
||||
_IsPermitted = true;
|
||||
|
||||
yield return new WaitForEndOfFrame();
|
||||
}
|
||||
}
|
||||
|
||||
public override void SelectSource(int sourceId)
|
||||
{
|
||||
if (sourceId < 0 || sourceId >= availableSources.Length)
|
||||
{
|
||||
throw new ArgumentException($"Invalid source ID: {sourceId}");
|
||||
}
|
||||
|
||||
webCamDevice = availableSources[sourceId];
|
||||
}
|
||||
|
||||
public override IEnumerator Play()
|
||||
{
|
||||
yield return new WaitUntil(() => _isInitialized);
|
||||
if (!_IsPermitted)
|
||||
{
|
||||
throw new InvalidOperationException("Not permitted to access cameras");
|
||||
}
|
||||
|
||||
InitializeWebCamTexture();
|
||||
webCamTexture.Play();
|
||||
yield return WaitForWebCamTexture();
|
||||
}
|
||||
|
||||
public override IEnumerator Resume()
|
||||
{
|
||||
if (!isPrepared)
|
||||
{
|
||||
throw new InvalidOperationException("WebCamTexture is not prepared yet");
|
||||
}
|
||||
if (!webCamTexture.isPlaying)
|
||||
{
|
||||
webCamTexture.Play();
|
||||
}
|
||||
yield return WaitForWebCamTexture();
|
||||
}
|
||||
|
||||
public override void Pause()
|
||||
{
|
||||
if (isPlaying)
|
||||
{
|
||||
webCamTexture.Pause();
|
||||
}
|
||||
}
|
||||
|
||||
public override void Stop()
|
||||
{
|
||||
if (webCamTexture != null)
|
||||
{
|
||||
webCamTexture.Stop();
|
||||
}
|
||||
webCamTexture = null;
|
||||
}
|
||||
|
||||
public override Texture GetCurrentTexture()
|
||||
{
|
||||
return webCamTexture;
|
||||
}
|
||||
|
||||
private ResolutionStruct GetDefaultResolution()
|
||||
{
|
||||
var resolutions = availableResolutions;
|
||||
return resolutions == null || resolutions.Length == 0 ? new ResolutionStruct() : resolutions.OrderBy(resolution => resolution, new ResolutionStructComparer(_preferableDefaultWidth)).First();
|
||||
}
|
||||
|
||||
private void InitializeWebCamTexture()
|
||||
{
|
||||
Stop();
|
||||
if (webCamDevice is WebCamDevice valueOfWebCamDevice)
|
||||
{
|
||||
webCamTexture = new WebCamTexture(valueOfWebCamDevice.name, resolution.width, resolution.height, (int)resolution.frameRate);
|
||||
return;
|
||||
}
|
||||
throw new InvalidOperationException("Cannot initialize WebCamTexture because WebCamDevice is not selected");
|
||||
}
|
||||
|
||||
private IEnumerator WaitForWebCamTexture()
|
||||
{
|
||||
const int timeoutFrame = 2000;
|
||||
var count = 0;
|
||||
Logger.LogVerbose("Waiting for WebCamTexture to start");
|
||||
yield return new WaitUntil(() => count++ > timeoutFrame || webCamTexture.width > 16);
|
||||
|
||||
if (webCamTexture.width <= 16)
|
||||
{
|
||||
throw new TimeoutException("Failed to start WebCam");
|
||||
}
|
||||
}
|
||||
|
||||
private class ResolutionStructComparer : IComparer<ResolutionStruct>
|
||||
{
|
||||
private readonly int _preferableDefaultWidth;
|
||||
|
||||
public ResolutionStructComparer(int preferableDefaultWidth)
|
||||
{
|
||||
_preferableDefaultWidth = preferableDefaultWidth;
|
||||
}
|
||||
|
||||
public int Compare(ResolutionStruct a, ResolutionStruct b)
|
||||
{
|
||||
var aDiff = Mathf.Abs(a.width - _preferableDefaultWidth);
|
||||
var bDiff = Mathf.Abs(b.width - _preferableDefaultWidth);
|
||||
if (aDiff != bDiff)
|
||||
{
|
||||
return aDiff - bDiff;
|
||||
}
|
||||
if (a.height != b.height)
|
||||
{
|
||||
// prefer smaller height
|
||||
return a.height - b.height;
|
||||
}
|
||||
// prefer smaller frame rate
|
||||
return (int)(a.frameRate - b.frameRate);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
fileFormatVersion: 2
|
||||
guid: 498146e99d4934673bd948c8be11227e
|
||||
MonoImporter:
|
||||
externalObjects: {}
|
||||
serializedVersion: 2
|
||||
defaultReferences: []
|
||||
executionOrder: 0
|
||||
icon: {instanceID: 0}
|
||||
userData:
|
||||
assetBundleName:
|
||||
assetBundleVariant:
|
||||
45
Assets/MediaPipeUnity/Common/Scripts/ImageSourceProvider.cs
Normal file
45
Assets/MediaPipeUnity/Common/Scripts/ImageSourceProvider.cs
Normal file
@@ -0,0 +1,45 @@
|
||||
// Copyright (c) 2021 homuler
|
||||
//
|
||||
// Use of this source code is governed by an MIT-style
|
||||
// license that can be found in the LICENSE file or at
|
||||
// https://opensource.org/licenses/MIT.
|
||||
|
||||
namespace Mediapipe.Unity
|
||||
{
|
||||
public static class ImageSourceProvider
|
||||
{
|
||||
private static ImageSource _ImageSource;
|
||||
public static ImageSource ImageSource
|
||||
{
|
||||
get => _ImageSource;
|
||||
set
|
||||
{
|
||||
if (value != null && !value.enabled)
|
||||
{
|
||||
value.enabled = true;
|
||||
}
|
||||
_ImageSource = value;
|
||||
}
|
||||
}
|
||||
|
||||
public static ImageSourceType CurrentSourceType
|
||||
{
|
||||
get
|
||||
{
|
||||
if (_ImageSource is WebCamSource)
|
||||
{
|
||||
return ImageSourceType.WebCamera;
|
||||
}
|
||||
if (_ImageSource is StaticImageSource)
|
||||
{
|
||||
return ImageSourceType.Image;
|
||||
}
|
||||
if (_ImageSource is VideoSource)
|
||||
{
|
||||
return ImageSourceType.Video;
|
||||
}
|
||||
return ImageSourceType.Unknown;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
fileFormatVersion: 2
|
||||
guid: 703949cb153f043aca7381a8f9b21a86
|
||||
MonoImporter:
|
||||
externalObjects: {}
|
||||
serializedVersion: 2
|
||||
defaultReferences: []
|
||||
executionOrder: 0
|
||||
icon: {instanceID: 0}
|
||||
userData:
|
||||
assetBundleName:
|
||||
assetBundleVariant:
|
||||
131
Assets/MediaPipeUnity/Common/Scripts/ImageSourceSolution.cs
Normal file
131
Assets/MediaPipeUnity/Common/Scripts/ImageSourceSolution.cs
Normal file
@@ -0,0 +1,131 @@
|
||||
// Copyright (c) 2021 homuler
|
||||
//
|
||||
// Use of this source code is governed by an MIT-style
|
||||
// license that can be found in the LICENSE file or at
|
||||
// https://opensource.org/licenses/MIT.
|
||||
|
||||
using System.Collections;
|
||||
using UnityEngine;
|
||||
|
||||
namespace Mediapipe.Unity
|
||||
{
|
||||
public abstract class ImageSourceSolution<T> : Solution where T : GraphRunner
|
||||
{
|
||||
[SerializeField] protected Screen screen;
|
||||
[SerializeField] protected T graphRunner;
|
||||
[SerializeField] protected TextureFramePool textureFramePool;
|
||||
|
||||
private Coroutine _coroutine;
|
||||
|
||||
public RunningMode runningMode;
|
||||
|
||||
public long timeoutMillisec
|
||||
{
|
||||
get => graphRunner.timeoutMillisec;
|
||||
set => graphRunner.timeoutMillisec = value;
|
||||
}
|
||||
|
||||
public override void Play()
|
||||
{
|
||||
if (_coroutine != null)
|
||||
{
|
||||
Stop();
|
||||
}
|
||||
base.Play();
|
||||
_coroutine = StartCoroutine(Run());
|
||||
}
|
||||
|
||||
public override void Pause()
|
||||
{
|
||||
base.Pause();
|
||||
ImageSourceProvider.ImageSource.Pause();
|
||||
}
|
||||
|
||||
public override void Resume()
|
||||
{
|
||||
base.Resume();
|
||||
var _ = StartCoroutine(ImageSourceProvider.ImageSource.Resume());
|
||||
}
|
||||
|
||||
public override void Stop()
|
||||
{
|
||||
base.Stop();
|
||||
StopCoroutine(_coroutine);
|
||||
ImageSourceProvider.ImageSource.Stop();
|
||||
graphRunner.Stop();
|
||||
}
|
||||
|
||||
private IEnumerator Run()
|
||||
{
|
||||
var graphInitRequest = graphRunner.WaitForInit(runningMode);
|
||||
var imageSource = ImageSourceProvider.ImageSource;
|
||||
|
||||
yield return imageSource.Play();
|
||||
|
||||
if (!imageSource.isPrepared)
|
||||
{
|
||||
Logger.LogError(TAG, "Failed to start ImageSource, exiting...");
|
||||
yield break;
|
||||
}
|
||||
|
||||
// Use RGBA32 as the input format.
|
||||
// TODO: When using GpuBuffer, MediaPipe assumes that the input format is BGRA, so the following code must be fixed.
|
||||
textureFramePool.ResizeTexture(imageSource.textureWidth, imageSource.textureHeight, TextureFormat.RGBA32);
|
||||
SetupScreen(imageSource);
|
||||
|
||||
yield return graphInitRequest;
|
||||
if (graphInitRequest.isError)
|
||||
{
|
||||
Logger.LogError(TAG, graphInitRequest.error);
|
||||
yield break;
|
||||
}
|
||||
|
||||
OnStartRun();
|
||||
graphRunner.StartRun(imageSource);
|
||||
|
||||
var waitWhilePausing = new WaitWhile(() => isPaused);
|
||||
|
||||
while (true)
|
||||
{
|
||||
if (isPaused)
|
||||
{
|
||||
yield return waitWhilePausing;
|
||||
}
|
||||
|
||||
if (!textureFramePool.TryGetTextureFrame(out var textureFrame))
|
||||
{
|
||||
yield return new WaitForEndOfFrame();
|
||||
continue;
|
||||
}
|
||||
|
||||
// Copy current image to TextureFrame
|
||||
ReadFromImageSource(imageSource, textureFrame);
|
||||
AddTextureFrameToInputStream(textureFrame);
|
||||
yield return new WaitForEndOfFrame();
|
||||
|
||||
if (runningMode.IsSynchronous())
|
||||
{
|
||||
RenderCurrentFrame(textureFrame);
|
||||
yield return WaitForNextValue();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
protected virtual void SetupScreen(ImageSource imageSource)
|
||||
{
|
||||
// NOTE: The screen will be resized later, keeping the aspect ratio.
|
||||
screen.Initialize(imageSource);
|
||||
}
|
||||
|
||||
protected virtual void RenderCurrentFrame(TextureFrame textureFrame)
|
||||
{
|
||||
screen.ReadSync(textureFrame);
|
||||
}
|
||||
|
||||
protected abstract void OnStartRun();
|
||||
|
||||
protected abstract void AddTextureFrameToInputStream(TextureFrame textureFrame);
|
||||
|
||||
protected abstract IEnumerator WaitForNextValue();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
fileFormatVersion: 2
|
||||
guid: a7573abd3c3f9972e964d5c526c34e0a
|
||||
MonoImporter:
|
||||
externalObjects: {}
|
||||
serializedVersion: 2
|
||||
defaultReferences: []
|
||||
executionOrder: 0
|
||||
icon: {instanceID: 0}
|
||||
userData:
|
||||
assetBundleName:
|
||||
assetBundleVariant:
|
||||
15
Assets/MediaPipeUnity/Common/Scripts/InferenceMode.cs
Normal file
15
Assets/MediaPipeUnity/Common/Scripts/InferenceMode.cs
Normal file
@@ -0,0 +1,15 @@
|
||||
// Copyright (c) 2021 homuler
|
||||
//
|
||||
// Use of this source code is governed by an MIT-style
|
||||
// license that can be found in the LICENSE file or at
|
||||
// https://opensource.org/licenses/MIT.
|
||||
|
||||
namespace Mediapipe.Unity
|
||||
{
|
||||
[System.Serializable]
|
||||
public enum InferenceMode
|
||||
{
|
||||
GPU,
|
||||
CPU,
|
||||
}
|
||||
}
|
||||
11
Assets/MediaPipeUnity/Common/Scripts/InferenceMode.cs.meta
Normal file
11
Assets/MediaPipeUnity/Common/Scripts/InferenceMode.cs.meta
Normal file
@@ -0,0 +1,11 @@
|
||||
fileFormatVersion: 2
|
||||
guid: d95cf1e3a8fe6741295a3b42972dec58
|
||||
MonoImporter:
|
||||
externalObjects: {}
|
||||
serializedVersion: 2
|
||||
defaultReferences: []
|
||||
executionOrder: 0
|
||||
icon: {instanceID: 0}
|
||||
userData:
|
||||
assetBundleName:
|
||||
assetBundleVariant:
|
||||
264
Assets/MediaPipeUnity/Common/Scripts/MemoizedLogger.cs
Normal file
264
Assets/MediaPipeUnity/Common/Scripts/MemoizedLogger.cs
Normal file
@@ -0,0 +1,264 @@
|
||||
// Copyright (c) 2021 homuler
|
||||
//
|
||||
// Use of this source code is governed by an MIT-style
|
||||
// license that can be found in the LICENSE file or at
|
||||
// https://opensource.org/licenses/MIT.
|
||||
|
||||
using System;
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
using UnityEngine;
|
||||
|
||||
using LogLevel = Mediapipe.Unity.Logger.LogLevel;
|
||||
|
||||
namespace Mediapipe.Unity
|
||||
{
|
||||
public class MemoizedLogger : IExtendedLogger
|
||||
{
|
||||
public readonly struct LogStruct
|
||||
{
|
||||
public readonly LogLevel logLevel;
|
||||
public readonly string tag;
|
||||
public readonly object message;
|
||||
public readonly DateTime utcTime;
|
||||
|
||||
public LogStruct(LogLevel logLevel, string tag, object message)
|
||||
{
|
||||
this.logLevel = logLevel;
|
||||
this.tag = tag;
|
||||
this.message = message;
|
||||
utcTime = DateTime.UtcNow;
|
||||
}
|
||||
|
||||
public LogStruct(LogType logType, string tag, object message) : this(GetLogLevelFromLogType(logType), tag, message) { }
|
||||
|
||||
private static LogLevel GetLogLevelFromLogType(LogType logType)
|
||||
{
|
||||
switch (logType)
|
||||
{
|
||||
case LogType.Error:
|
||||
case LogType.Exception:
|
||||
{
|
||||
return LogLevel.Error;
|
||||
}
|
||||
case LogType.Warning:
|
||||
{
|
||||
return LogLevel.Warn;
|
||||
}
|
||||
case LogType.Assert:
|
||||
case LogType.Log:
|
||||
default:
|
||||
{
|
||||
return LogLevel.Info;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public MemoizedLogger(int historySize = 0)
|
||||
{
|
||||
this.historySize = historySize;
|
||||
}
|
||||
|
||||
private int _historySize;
|
||||
public int historySize
|
||||
{
|
||||
get => _historySize;
|
||||
set
|
||||
{
|
||||
_historySize = value;
|
||||
|
||||
while (_historySize < histories.Count)
|
||||
{
|
||||
var _ = histories.Dequeue();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private Queue<LogStruct> _histories;
|
||||
public Queue<LogStruct> histories
|
||||
{
|
||||
get
|
||||
{
|
||||
if (_histories == null)
|
||||
{
|
||||
_histories = new Queue<LogStruct>(_historySize);
|
||||
}
|
||||
return _histories;
|
||||
}
|
||||
}
|
||||
|
||||
public delegate void LogOutputEventHandler(LogStruct logStruct);
|
||||
public event LogOutputEventHandler OnLogOutput;
|
||||
|
||||
private readonly ILogger _logger = Debug.unityLogger;
|
||||
|
||||
public LogType filterLogType
|
||||
{
|
||||
get => _logger.filterLogType;
|
||||
set => _logger.filterLogType = value;
|
||||
}
|
||||
|
||||
public bool logEnabled
|
||||
{
|
||||
get => _logger.logEnabled;
|
||||
set => _logger.logEnabled = value;
|
||||
}
|
||||
|
||||
public ILogHandler logHandler
|
||||
{
|
||||
get => _logger.logHandler;
|
||||
set => _logger.logHandler = value;
|
||||
}
|
||||
|
||||
public bool IsLogTypeAllowed(LogType logType)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
public void Log(object message)
|
||||
{
|
||||
_logger.Log(message);
|
||||
RecordInfoLog(null, message);
|
||||
}
|
||||
|
||||
public void Log(string tag, object message)
|
||||
{
|
||||
_logger.Log(tag, message);
|
||||
RecordInfoLog(tag, message);
|
||||
}
|
||||
|
||||
public void Log(string tag, object message, UnityEngine.Object context)
|
||||
{
|
||||
_logger.Log(tag, message, context);
|
||||
RecordInfoLog(tag, message);
|
||||
}
|
||||
|
||||
public void Log(LogType logType, object message)
|
||||
{
|
||||
_logger.Log(logType, message);
|
||||
RecordLog(logType, null, message);
|
||||
}
|
||||
|
||||
public void Log(LogType logType, object message, UnityEngine.Object context)
|
||||
{
|
||||
_logger.Log(logType, message, context);
|
||||
RecordLog(logType, null, message);
|
||||
}
|
||||
|
||||
public void Log(LogType logType, string tag, object message)
|
||||
{
|
||||
_logger.Log(logType, tag, message);
|
||||
RecordLog(logType, tag, message);
|
||||
}
|
||||
|
||||
public void Log(LogType logType, string tag, object message, UnityEngine.Object context)
|
||||
{
|
||||
_logger.Log(logType, tag, message, context);
|
||||
RecordLog(logType, tag, message);
|
||||
}
|
||||
|
||||
public void Log(LogLevel logLevel, string tag, object message, UnityEngine.Object context)
|
||||
{
|
||||
_logger.Log(logLevel.GetLogType(), tag, message, context);
|
||||
RecordLog(new LogStruct(logLevel, tag, message));
|
||||
}
|
||||
|
||||
public void Log(LogLevel logLevel, string tag, object message)
|
||||
{
|
||||
_logger.Log(logLevel.GetLogType(), tag, message);
|
||||
RecordLog(new LogStruct(logLevel, tag, message));
|
||||
}
|
||||
|
||||
public void Log(LogLevel logLevel, object message, UnityEngine.Object context)
|
||||
{
|
||||
_logger.Log(logLevel.GetLogType(), message, context);
|
||||
RecordLog(new LogStruct(logLevel, null, message));
|
||||
}
|
||||
|
||||
public void Log(LogLevel logLevel, object message)
|
||||
{
|
||||
_logger.Log(logLevel.GetLogType(), message);
|
||||
RecordLog(new LogStruct(logLevel, null, message));
|
||||
}
|
||||
|
||||
public void LogWarning(string tag, object message)
|
||||
{
|
||||
_logger.LogWarning(tag, message);
|
||||
RecordWarnLog(tag, message);
|
||||
}
|
||||
|
||||
public void LogWarning(string tag, object message, UnityEngine.Object context)
|
||||
{
|
||||
_logger.LogWarning(tag, message, context);
|
||||
RecordWarnLog(tag, message);
|
||||
}
|
||||
|
||||
public void LogError(string tag, object message)
|
||||
{
|
||||
_logger.LogError(tag, message);
|
||||
RecordErrorLog(tag, message);
|
||||
}
|
||||
|
||||
public void LogError(string tag, object message, UnityEngine.Object context)
|
||||
{
|
||||
_logger.LogError(tag, message, context);
|
||||
RecordErrorLog(tag, message);
|
||||
}
|
||||
|
||||
public void LogFormat(LogType logType, string format, params object[] args)
|
||||
{
|
||||
_logger.LogFormat(logType, format, args);
|
||||
}
|
||||
|
||||
public void LogFormat(LogType logType, UnityEngine.Object context, string format, params object[] args)
|
||||
{
|
||||
_logger.LogFormat(logType, context, format, args);
|
||||
}
|
||||
|
||||
public void LogException(Exception exception)
|
||||
{
|
||||
_logger.LogException(exception);
|
||||
RecordErrorLog(null, exception);
|
||||
}
|
||||
|
||||
public void LogException(Exception exception, UnityEngine.Object context)
|
||||
{
|
||||
_logger.LogException(exception, context);
|
||||
RecordErrorLog(null, exception);
|
||||
}
|
||||
|
||||
public void RecordLog(LogStruct log)
|
||||
{
|
||||
lock (((ICollection)histories).SyncRoot)
|
||||
{
|
||||
while (histories.Count > 0 && _historySize <= histories.Count)
|
||||
{
|
||||
var _ = histories.Dequeue();
|
||||
}
|
||||
histories.Enqueue(log);
|
||||
OnLogOutput?.Invoke(log);
|
||||
}
|
||||
}
|
||||
|
||||
private void RecordLog(LogType logType, string tag, object message)
|
||||
{
|
||||
RecordLog(new LogStruct(logType, tag, message));
|
||||
}
|
||||
|
||||
private void RecordInfoLog(string tag, object message)
|
||||
{
|
||||
RecordLog(new LogStruct(LogLevel.Info, tag, message));
|
||||
}
|
||||
|
||||
private void RecordWarnLog(string tag, object message)
|
||||
{
|
||||
RecordLog(new LogStruct(LogLevel.Warn, tag, message));
|
||||
}
|
||||
|
||||
private void RecordErrorLog(string tag, object message)
|
||||
{
|
||||
RecordLog(new LogStruct(LogLevel.Error, tag, message));
|
||||
}
|
||||
}
|
||||
}
|
||||
11
Assets/MediaPipeUnity/Common/Scripts/MemoizedLogger.cs.meta
Normal file
11
Assets/MediaPipeUnity/Common/Scripts/MemoizedLogger.cs.meta
Normal file
@@ -0,0 +1,11 @@
|
||||
fileFormatVersion: 2
|
||||
guid: 00164446b902a4f99ae323de716782fe
|
||||
MonoImporter:
|
||||
externalObjects: {}
|
||||
serializedVersion: 2
|
||||
defaultReferences: []
|
||||
executionOrder: 0
|
||||
icon: {instanceID: 0}
|
||||
userData:
|
||||
assetBundleName:
|
||||
assetBundleVariant:
|
||||
24
Assets/MediaPipeUnity/Common/Scripts/RunningMode.cs
Normal file
24
Assets/MediaPipeUnity/Common/Scripts/RunningMode.cs
Normal file
@@ -0,0 +1,24 @@
|
||||
// Copyright (c) 2021 homuler
|
||||
//
|
||||
// Use of this source code is governed by an MIT-style
|
||||
// license that can be found in the LICENSE file or at
|
||||
// https://opensource.org/licenses/MIT.
|
||||
|
||||
namespace Mediapipe.Unity
|
||||
{
|
||||
[System.Serializable]
|
||||
public enum RunningMode
|
||||
{
|
||||
Async,
|
||||
NonBlockingSync,
|
||||
Sync,
|
||||
}
|
||||
|
||||
public static class RunningModeExtension
|
||||
{
|
||||
public static bool IsSynchronous(this RunningMode runningMode)
|
||||
{
|
||||
return runningMode == RunningMode.Sync || runningMode == RunningMode.NonBlockingSync;
|
||||
}
|
||||
}
|
||||
}
|
||||
11
Assets/MediaPipeUnity/Common/Scripts/RunningMode.cs.meta
Normal file
11
Assets/MediaPipeUnity/Common/Scripts/RunningMode.cs.meta
Normal file
@@ -0,0 +1,11 @@
|
||||
fileFormatVersion: 2
|
||||
guid: e82590f067b6f1b96ac9dadddcd9ab26
|
||||
MonoImporter:
|
||||
externalObjects: {}
|
||||
serializedVersion: 2
|
||||
defaultReferences: []
|
||||
executionOrder: 0
|
||||
icon: {instanceID: 0}
|
||||
userData:
|
||||
assetBundleName:
|
||||
assetBundleVariant:
|
||||
98
Assets/MediaPipeUnity/Common/Scripts/Screen.cs
Normal file
98
Assets/MediaPipeUnity/Common/Scripts/Screen.cs
Normal file
@@ -0,0 +1,98 @@
|
||||
// Copyright (c) 2021 homuler
|
||||
//
|
||||
// Use of this source code is governed by an MIT-style
|
||||
// license that can be found in the LICENSE file or at
|
||||
// https://opensource.org/licenses/MIT.
|
||||
|
||||
using UnityEngine;
|
||||
using UnityEngine.UI;
|
||||
|
||||
namespace Mediapipe.Unity
|
||||
{
|
||||
public class Screen : MonoBehaviour
|
||||
{
|
||||
[SerializeField] private RawImage _screen;
|
||||
|
||||
private ImageSource _imageSource;
|
||||
|
||||
public Texture texture
|
||||
{
|
||||
private get => _screen.texture;
|
||||
set => _screen.texture = value;
|
||||
}
|
||||
|
||||
public UnityEngine.Rect uvRect
|
||||
{
|
||||
set => _screen.uvRect = value;
|
||||
}
|
||||
|
||||
public void Initialize(ImageSource imageSource)
|
||||
{
|
||||
_imageSource = imageSource;
|
||||
|
||||
Resize(_imageSource.textureWidth, _imageSource.textureHeight);
|
||||
Rotate(_imageSource.rotation.Reverse());
|
||||
ResetUvRect(RunningMode.Async);
|
||||
texture = imageSource.GetCurrentTexture();
|
||||
}
|
||||
|
||||
public void Resize(int width, int height)
|
||||
{
|
||||
_screen.rectTransform.sizeDelta = new Vector2(width, height);
|
||||
}
|
||||
|
||||
public void Rotate(RotationAngle rotationAngle)
|
||||
{
|
||||
_screen.rectTransform.localEulerAngles = rotationAngle.GetEulerAngles();
|
||||
}
|
||||
|
||||
public void ReadSync(TextureFrame textureFrame)
|
||||
{
|
||||
if (!(texture is Texture2D))
|
||||
{
|
||||
texture = new Texture2D(_imageSource.textureWidth, _imageSource.textureHeight, TextureFormat.RGBA32, false);
|
||||
ResetUvRect(RunningMode.Sync);
|
||||
}
|
||||
textureFrame.CopyTexture(texture);
|
||||
}
|
||||
|
||||
private void ResetUvRect(RunningMode runningMode)
|
||||
{
|
||||
var rect = new UnityEngine.Rect(0, 0, 1, 1);
|
||||
|
||||
if (_imageSource.isVerticallyFlipped && runningMode == RunningMode.Async)
|
||||
{
|
||||
// In Async mode, we don't need to flip the screen vertically since the image will be copied on CPU.
|
||||
rect = FlipVertically(rect);
|
||||
}
|
||||
|
||||
if (_imageSource.isFrontFacing)
|
||||
{
|
||||
// Flip the image (not the screen) horizontally.
|
||||
// It should be taken into account that the image will be rotated later.
|
||||
var rotation = _imageSource.rotation;
|
||||
|
||||
if (rotation == RotationAngle.Rotation0 || rotation == RotationAngle.Rotation180)
|
||||
{
|
||||
rect = FlipHorizontally(rect);
|
||||
}
|
||||
else
|
||||
{
|
||||
rect = FlipVertically(rect);
|
||||
}
|
||||
}
|
||||
|
||||
uvRect = rect;
|
||||
}
|
||||
|
||||
private UnityEngine.Rect FlipHorizontally(UnityEngine.Rect rect)
|
||||
{
|
||||
return new UnityEngine.Rect(1 - rect.x, rect.y, -rect.width, rect.height);
|
||||
}
|
||||
|
||||
private UnityEngine.Rect FlipVertically(UnityEngine.Rect rect)
|
||||
{
|
||||
return new UnityEngine.Rect(rect.x, 1 - rect.y, rect.width, -rect.height);
|
||||
}
|
||||
}
|
||||
}
|
||||
11
Assets/MediaPipeUnity/Common/Scripts/Screen.cs.meta
Normal file
11
Assets/MediaPipeUnity/Common/Scripts/Screen.cs.meta
Normal file
@@ -0,0 +1,11 @@
|
||||
fileFormatVersion: 2
|
||||
guid: 827c4431af677e057aa6f14170d0785c
|
||||
MonoImporter:
|
||||
externalObjects: {}
|
||||
serializedVersion: 2
|
||||
defaultReferences: []
|
||||
executionOrder: 0
|
||||
icon: {instanceID: 0}
|
||||
userData:
|
||||
assetBundleName:
|
||||
assetBundleVariant:
|
||||
117
Assets/MediaPipeUnity/Common/Scripts/Solution.cs
Normal file
117
Assets/MediaPipeUnity/Common/Scripts/Solution.cs
Normal file
@@ -0,0 +1,117 @@
|
||||
// Copyright (c) 2021 homuler
|
||||
//
|
||||
// Use of this source code is governed by an MIT-style
|
||||
// license that can be found in the LICENSE file or at
|
||||
// https://opensource.org/licenses/MIT.
|
||||
|
||||
using System.Collections;
|
||||
using UnityEngine;
|
||||
|
||||
namespace Mediapipe.Unity
|
||||
{
|
||||
public abstract class Solution : MonoBehaviour
|
||||
{
|
||||
#pragma warning disable IDE1006
|
||||
// TODO: make it static
|
||||
protected virtual string TAG => GetType().Name;
|
||||
#pragma warning restore IDE1006
|
||||
|
||||
public Bootstrap bootstrap;
|
||||
protected bool isPaused;
|
||||
|
||||
protected virtual IEnumerator Start()
|
||||
{
|
||||
bootstrap = FindBootstrap();
|
||||
yield return new WaitUntil(() => bootstrap.isFinished);
|
||||
|
||||
Play();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Start the main program from the beginning.
|
||||
/// </summary>
|
||||
public virtual void Play()
|
||||
{
|
||||
isPaused = false;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Pause the main program.
|
||||
/// <summary>
|
||||
public virtual void Pause()
|
||||
{
|
||||
isPaused = true;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Resume the main program.
|
||||
/// If the main program has not begun, it'll do nothing.
|
||||
/// </summary>
|
||||
public virtual void Resume()
|
||||
{
|
||||
isPaused = false;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Stops the main program.
|
||||
/// </summary>
|
||||
public virtual void Stop()
|
||||
{
|
||||
isPaused = true;
|
||||
}
|
||||
|
||||
protected static void SetupAnnotationController<T>(AnnotationController<T> annotationController, ImageSource imageSource, bool expectedToBeMirrored = false) where T : HierarchicalAnnotation
|
||||
{
|
||||
annotationController.isMirrored = expectedToBeMirrored ^ imageSource.isHorizontallyFlipped ^ imageSource.isFrontFacing;
|
||||
annotationController.rotationAngle = imageSource.rotation.Reverse();
|
||||
}
|
||||
|
||||
protected static void ReadFromImageSource(ImageSource imageSource, TextureFrame textureFrame)
|
||||
{
|
||||
var sourceTexture = imageSource.GetCurrentTexture();
|
||||
|
||||
// For some reason, when the image is coiped on GPU, latency tends to be high.
|
||||
// So even when OpenGL ES is available, use CPU to copy images.
|
||||
var textureType = sourceTexture.GetType();
|
||||
|
||||
if (textureType == typeof(WebCamTexture))
|
||||
{
|
||||
textureFrame.ReadTextureFromOnCPU((WebCamTexture)sourceTexture);
|
||||
}
|
||||
else if (textureType == typeof(Texture2D))
|
||||
{
|
||||
textureFrame.ReadTextureFromOnCPU((Texture2D)sourceTexture);
|
||||
}
|
||||
else
|
||||
{
|
||||
textureFrame.ReadTextureFromOnCPU(sourceTexture);
|
||||
}
|
||||
}
|
||||
|
||||
protected Bootstrap FindBootstrap()
|
||||
{
|
||||
var bootstrapObj = GameObject.Find("Bootstrap");
|
||||
|
||||
if (bootstrapObj != null)
|
||||
{
|
||||
return bootstrapObj.GetComponent<Bootstrap>();
|
||||
}
|
||||
|
||||
Logger.LogWarning(TAG, "Global Bootstrap instance is not found (maybe running a sample scene directly), "
|
||||
+ "so activating a fallback Bootstrap instance attached to each Solution object");
|
||||
|
||||
var bootstrap = GetComponent<Bootstrap>();
|
||||
bootstrap.enabled = true;
|
||||
|
||||
// hide menu button when trying a single scene.
|
||||
DisableMenuButton();
|
||||
return bootstrap;
|
||||
}
|
||||
|
||||
private void DisableMenuButton()
|
||||
{
|
||||
var menuButton = GameObject.Find("MenuButton");
|
||||
menuButton.SetActive(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
11
Assets/MediaPipeUnity/Common/Scripts/Solution.cs.meta
Normal file
11
Assets/MediaPipeUnity/Common/Scripts/Solution.cs.meta
Normal file
@@ -0,0 +1,11 @@
|
||||
fileFormatVersion: 2
|
||||
guid: 589adc9f9488f9d8eaee5a408719b452
|
||||
MonoImporter:
|
||||
externalObjects: {}
|
||||
serializedVersion: 2
|
||||
defaultReferences: []
|
||||
executionOrder: 0
|
||||
icon: {instanceID: 0}
|
||||
userData:
|
||||
assetBundleName:
|
||||
assetBundleVariant:
|
||||
36
Assets/MediaPipeUnity/Common/Scripts/StartSceneController.cs
Normal file
36
Assets/MediaPipeUnity/Common/Scripts/StartSceneController.cs
Normal file
@@ -0,0 +1,36 @@
|
||||
// Copyright (c) 2021 homuler
|
||||
//
|
||||
// Use of this source code is governed by an MIT-style
|
||||
// license that can be found in the LICENSE file or at
|
||||
// https://opensource.org/licenses/MIT.
|
||||
|
||||
using System.Collections;
|
||||
using UnityEngine;
|
||||
using UnityEngine.UI;
|
||||
using UnityEngine.SceneManagement;
|
||||
|
||||
namespace Mediapipe.Unity
|
||||
{
|
||||
public class StartSceneController : MonoBehaviour
|
||||
{
|
||||
private const string _TAG = nameof(Bootstrap);
|
||||
|
||||
[SerializeField] private Image _screen;
|
||||
[SerializeField] private GameObject _consolePrefab;
|
||||
|
||||
private IEnumerator Start()
|
||||
{
|
||||
var _ = Instantiate(_consolePrefab, _screen.transform);
|
||||
|
||||
var bootstrap = GetComponent<Bootstrap>();
|
||||
|
||||
yield return new WaitUntil(() => bootstrap.isFinished);
|
||||
|
||||
DontDestroyOnLoad(gameObject);
|
||||
|
||||
Logger.LogInfo(_TAG, "Loading the first scene...");
|
||||
var sceneLoadReq = SceneManager.LoadSceneAsync(1);
|
||||
yield return new WaitUntil(() => sceneLoadReq.isDone);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
fileFormatVersion: 2
|
||||
guid: 2ca6b5afaa693af02957f0d68cf058d9
|
||||
MonoImporter:
|
||||
externalObjects: {}
|
||||
serializedVersion: 2
|
||||
defaultReferences: []
|
||||
executionOrder: 0
|
||||
icon: {instanceID: 0}
|
||||
userData:
|
||||
assetBundleName:
|
||||
assetBundleVariant:
|
||||
87
Assets/MediaPipeUnity/Common/Scripts/WaitForResult.cs
Normal file
87
Assets/MediaPipeUnity/Common/Scripts/WaitForResult.cs
Normal file
@@ -0,0 +1,87 @@
|
||||
// Copyright (c) 2021 homuler
|
||||
//
|
||||
// Use of this source code is governed by an MIT-style
|
||||
// license that can be found in the LICENSE file or at
|
||||
// https://opensource.org/licenses/MIT.
|
||||
|
||||
using System;
|
||||
using System.Collections;
|
||||
using UnityEngine;
|
||||
|
||||
using Stopwatch = System.Diagnostics.Stopwatch;
|
||||
|
||||
namespace Mediapipe.Unity
|
||||
{
|
||||
public class WaitForResult : CustomYieldInstruction
|
||||
{
|
||||
public object result { get; private set; }
|
||||
|
||||
protected object tmpResult;
|
||||
protected bool isDone = false;
|
||||
|
||||
private readonly MonoBehaviour _runner;
|
||||
private readonly IEnumerator _inner;
|
||||
private readonly Coroutine _coroutine;
|
||||
|
||||
public bool isError { get; private set; } = false;
|
||||
public Exception error { get; private set; }
|
||||
public override bool keepWaiting => !isDone && !isError;
|
||||
|
||||
public WaitForResult(MonoBehaviour runner, IEnumerator inner, long timeoutMillisec = long.MaxValue)
|
||||
{
|
||||
_runner = runner;
|
||||
_inner = inner;
|
||||
_coroutine = runner.StartCoroutine(Run(timeoutMillisec));
|
||||
}
|
||||
|
||||
private IEnumerator Run(long timeoutMillisec)
|
||||
{
|
||||
var stopwatch = new Stopwatch();
|
||||
stopwatch.Start();
|
||||
|
||||
while (true)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (stopwatch.ElapsedMilliseconds > timeoutMillisec)
|
||||
{
|
||||
_runner.StopCoroutine(_coroutine);
|
||||
throw new TimeoutException($"{stopwatch.ElapsedMilliseconds}ms has passed");
|
||||
}
|
||||
if (!_inner.MoveNext())
|
||||
{
|
||||
break;
|
||||
}
|
||||
tmpResult = _inner.Current;
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
isError = true;
|
||||
error = e;
|
||||
yield break;
|
||||
}
|
||||
yield return tmpResult;
|
||||
}
|
||||
Done(tmpResult);
|
||||
}
|
||||
|
||||
protected virtual void Done(object result)
|
||||
{
|
||||
this.result = result;
|
||||
isDone = true;
|
||||
}
|
||||
}
|
||||
|
||||
public class WaitForResult<T> : WaitForResult
|
||||
{
|
||||
public new T result { get; private set; }
|
||||
|
||||
public WaitForResult(MonoBehaviour runner, IEnumerator inner, long timeoutMillisec = long.MaxValue) : base(runner, inner, timeoutMillisec) { }
|
||||
|
||||
protected override void Done(object result)
|
||||
{
|
||||
this.result = (T)result;
|
||||
isDone = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
11
Assets/MediaPipeUnity/Common/Scripts/WaitForResult.cs.meta
Normal file
11
Assets/MediaPipeUnity/Common/Scripts/WaitForResult.cs.meta
Normal file
@@ -0,0 +1,11 @@
|
||||
fileFormatVersion: 2
|
||||
guid: 374b5ec183b8225d0ac8b0664812f5f1
|
||||
MonoImporter:
|
||||
externalObjects: {}
|
||||
serializedVersion: 2
|
||||
defaultReferences: []
|
||||
executionOrder: 0
|
||||
icon: {instanceID: 0}
|
||||
userData:
|
||||
assetBundleName:
|
||||
assetBundleVariant:
|
||||
Reference in New Issue
Block a user