using DigitalRuby.Tween;
using System;
using System.Collections;
using System.Collections.Generic;
using System.Linq;
using TMPro;
using UnityEngine;
using UnityEngine.UI;
using UnityEngine.Video;
///
/// TemplateCourse scene manager
///
public class CoursesController : AbstractFeedback
{
///
/// Reference to the objet holding the title
///
public TMP_Text courseTitle;
///
/// The current user
///
private User user;
///
/// Current user progress for this course
///
private PersistentDataController.SavedCourseProgress progress = null;
///
/// ScriptableObject with list of all courses
///
public CourseList courselist;
///
/// Reference to Course ScriptableObject
///
private Course course;
///
/// Index of the current word/letter in the course.learnables list
///
private int currentWordIndex = 0;
///
/// This holds the amount of words in the course
///
private int maxWords;
///
/// The "finished" screen
///
public GameObject ResultPanel;
///
/// Reference to the title on the results panel
///
public TMP_Text ResultsTitle;
///
/// Reference to the description on the results panel
///
public TMP_Text ResultsDecription;
///
/// Button to go back to courses list
///
public Button CoursesButton;
///
/// DateTime containint the start moment
///
private DateTime startMoment;
///
/// Reference to the timeSpent UI
///
public TMP_Text timeSpent;
///
/// Reference to the feedback field
///
private TMP_Text feedbackText;
///
/// Reference to the progress bar
///
private Slider feedbackProgress;
///
/// Reference to the progress bar image, so we can add fancy colors
///
private Image feedbackProgressImage;
///
/// Reference to the video player
///
public VideoPlayer videoPlayer;
///
/// Timer to keep track of how long a incorrect sign is performed
///
protected DateTime timer;
///
/// Current predicted sign
///
protected string predictedSign = null;
///
/// Previous incorrect sign, so we can keep track whether the user is wrong or the user is still changing signs
///
protected string previousIncorrectSign = null;
///
/// Keeps track of what type of panel is currently being used
///
protected int panelId = 0;
///
/// Boolean used to check whether the user has already answered the question
///
private bool hasAnswered = false;
///
/// Boolean used to check whether SlideIn animation is playing
///
private bool isNextSignInTransit = false;
///
/// Reference to course progress bar
///
public SlicedSlider progressBar;
///
/// Reference to the animator of the confetti animation
///
public Animator confettiAnimation;
///
/// Panel with video&image prefab
///
public GameObject panelSignWithVideoAndImagePrefab;
///
/// Panel with image prefab
///
public GameObject panelSignWithImagePrefab;
///
/// Panel with multiplechoice prefab
///
public GameObject panelMultipleChoicePrefab;
///
/// Reference to the canvas to put the panels into
///
public Transform canvas;
///
/// Reference to the previous panel,
/// so it can be deleted when its done playing its exit animation
///
private GameObject previousPanel = null;
///
/// Boolean used to make a test possible
///
private bool corruptPanelId = false;
///
/// Corrupted PanelID Value
///
private int CorruptedPanelIDValue = 999;
///
/// This function is called when the script is initialised.
/// It inactivatis the popup, finds a webcam to use and links it via the WebcamTexture to the display RawImage.
/// It takes the correct course from the courselist, using the courseIndex.
/// Then it checks whether or not the User has started the course yet, to possibly create a new progress atribute for the course.
/// Then it sets up the course-screen to display relevant information from the course-scriptable.
///
void Start()
{
StartCourseController();
signPredictor.SetSignsList(GetSignsList());
signPredictor.SetModel(course.theme.modelIndex);
AddSelfAsListener();
}
///
/// Fetches all the strings of the signs of the course
///
/// The signsList that needs to be passed to the signPredictor
private List GetSignsList()
{
List signsList = new List();
foreach (Learnable learnable in course.theme.learnables)
{
signsList.Add(learnable.name);
}
return signsList;
}
///
/// Holds the course-specific logic to start the controller, it is seperated to allow the course to be reset (if that would become needed)
///
public void StartCourseController()
{
// Setting up course
course = courselist.courses[courselist.currentCourseIndex];
maxWords = course.theme.learnables.Count;
// Reload from disk (course may be reset)
PersistentDataController.GetInstance().Load();
// Create entry in current user for keeping track of progress
user = UserList.GetCurrentUser();
progress = user.GetCourseProgress(course.index);
if (progress == null)
{
progress = new PersistentDataController.SavedCourseProgress();
progress.courseIndex = course.index;
int index = 0;
foreach (Learnable learnable in course.theme.learnables)
{
progress.AddLearnable(learnable.name, index++);
}
user.AddCourseProgress(progress);
}
UserList.Save();
progressBar.fillAmount = progress.progress;
currentWordIndex = 0;
previousPanel = SetupPanel();
// Hide the result panel
ResultPanel.SetActive(false);
// Set the startTime
startMoment = DateTime.Now;
}
///
/// Fetch the next sign and its panel type
///
/// A tuple of {next sign index, panel type}
///
/// The different panel types:
/// 0 : panelSignWithVideoAndImagePrefab
/// 1 : panelMultipleChoicePrefab
/// 2 : panelSignWithImagePrefab
///
private Tuple FetchSign()
{
PersistentDataController.SavedLearnableProgress learnable = progress.GetRandomLearnable();
int panelChosen;
if (learnable.progress > 2.0f)
{
panelChosen = 2;
}
else if (learnable.progress > 1.0f)
{
panelChosen = 1;
}
else
{
panelChosen = 0;
}
return Tuple.Create(learnable.index, panelChosen);
}
///
/// This function is called when the next-sign button is pressed.
/// It increased the wordindex and fetches new videos/images if index
public void NextSign()
{
// This function is also called (async) when pressing the 'Gebaar overslaan' button,
// so check for condition so we don't skip multiple signs
//if (isNextSignInTransit || maxWords < progress.completedLearnables)
if (isNextSignInTransit) return;
// Code for preview-progress, skipping should give progress unless it is multipleChoice
if (course.theme.modelIndex == ModelIndex.NONE)
{
string currentName = course.theme.learnables[currentWordIndex].name;
// This works both to allow panel 0 to allow progress via skipping and also to allow panel 2 to be skipped.
if (progress.FindLearnable(currentName).progress <= 1f || progress.FindLearnable(currentName).progress >= 2f) progress.UpdateLearnable(currentName, 1.5f);
}
progress.progress = (float)progress.completedLearnables / (float)maxWords;
progressBar.fillAmount = progress.progress;
// Update UI if course is not finished yet
if (progress.completedLearnables < maxWords)
{
// Set next sign/video/image
StartCoroutine(CRNextSign());
}
// Finish course and record progress
else
{
FinishCourse();
}
}
///
/// Coroutine for going to the next sign
///
///
private IEnumerator CRNextSign()
{
isNextSignInTransit = true;
GameObject newPanel = SetupPanel();
previousPanel.transform.SetAsFirstSibling();
newPanel.GetComponent().SetTrigger("Slide Panel In");
if (previousPanel != null)
previousPanel.GetComponent().SetTrigger("Slide Panel Out");
yield return new WaitForSeconds(1.0f);
confettiAnimation.ResetTrigger("Display Confetti");
GameObject.Destroy(previousPanel);
previousPanel = newPanel;
hasAnswered = false;
isNextSignInTransit = false;
}
///
/// Setup a new panel
///
/// Reference to the GameObject of the panel
private GameObject SetupPanel()
{
if (corruptPanelId == true)
{
(currentWordIndex, panelId) = (1, CorruptedPanelIDValue);
}
else
{
(currentWordIndex, panelId) = FetchSign().ToValueTuple();
}
switch (panelId)
{
case 0:
{
GameObject panel = GameObject.Instantiate(panelSignWithVideoAndImagePrefab, canvas);
panel.transform.SetAsFirstSibling();
PanelWithVideoAndImage script = panel.GetComponent();
script.signs = course.theme.learnables;
script.currentSignIndex = currentWordIndex;
script.isPreview = (course.theme.modelIndex == ModelIndex.NONE);
script.videoPlayer = videoPlayer;
feedbackProgress = script.feedbackProgressBar;
feedbackProgressImage = script.feedbackProgressImage;
feedbackText = script.feedbackText;
script.Display();
signPredictor.SwapScreen(script.webcamScreen);
courseTitle.text = "Voer het gebaar uit voor \"" + course.theme.learnables[currentWordIndex].name + "\"";
return panel;
}
case 1:
{
GameObject panel = GameObject.Instantiate(panelMultipleChoicePrefab, canvas);
panel.transform.SetAsFirstSibling();
PanelMultipleChoice script = panel.GetComponent();
script.signs = course.theme.learnables;
script.currentSignIndex = currentWordIndex;
script.videoPlayer = videoPlayer;
script.courseController = this;
script.progress = progress;
script.isFingerSpelling = course.theme.title == "Handalfabet";
script.Display();
signPredictor.SwapScreen(script.webcamScreen);
courseTitle.text = "Welk gebaar zie je hier?";
return panel;
}
case 2:
{
GameObject panel = GameObject.Instantiate(panelSignWithImagePrefab, canvas);
panel.transform.SetAsFirstSibling();
PanelWithImage script = panel.GetComponent();
script.signs = course.theme.learnables;
script.currentSignIndex = currentWordIndex;
script.isPreview = (course.theme.modelIndex == ModelIndex.NONE);
feedbackProgress = script.feedbackProgressBar;
feedbackProgressImage = script.feedbackProgressImage;
feedbackText = script.feedbackText;
script.Display();
signPredictor.SwapScreen(script.webcamScreen);
courseTitle.text = "Voer het gebaar uit voor \"" + course.theme.learnables[currentWordIndex].name + "\"";
return panel;
}
}
return null;
}
///
/// finishcourse is called to save the "finished" progress to the user.
///
public void FinishCourse()
{
// Show the "finished" screen
ResultPanel.SetActive(true);
// Set the correct title
ResultsTitle.text = course.title + " is voltooid!";
// Set the correct description
ResultsDecription.text = "Goed gedaan! Je kan nu spelletjes spelen met " + course.title + " om verder te oefenen!";
// Set the total time spent UI
TimeSpan time = DateTime.Now - startMoment;
timeSpent.text = time.ToString(@"hh\:mm\:ss");
// Link button
CoursesButton.onClick.AddListener(() => { SystemController.GetInstance().BackToPreviousScene(); });
progress.progress = 1.0f;
UserList.Save();
}
///
/// The updateFunction that is called when new probabilities become available
///
///
protected override IEnumerator UpdateFeedback()
{
// Check if the current word index is still in bounds, and if the current panel type is not multiple choice
if (currentWordIndex < course.theme.learnables.Count && panelId != 1 && !hasAnswered)
{
// Get current sign
Learnable sign = course.theme.learnables[currentWordIndex];
string currentSign = sign.name.ToUpper().Replace(" ", "-");
// Get the predicted sign
if (signPredictor != null && signPredictor.learnableProbabilities != null &&
currentSign != null && signPredictor.learnableProbabilities.ContainsKey(currentSign))
{
//Debug.Log($"{signPredictor.learnableProbabilities.Aggregate("", (t, e) => $"{t}{e.Key}={e.Value}, ")}");
float distCurrentSign = signPredictor.learnableProbabilities[currentSign];
// Get highest predicted sign
string predictedSign = signPredictor.learnableProbabilities.Aggregate((a, b) => a.Value < b.Value ? a : b).Key;
float distPredictSign = signPredictor.learnableProbabilities[predictedSign];
Learnable predSign = course.theme.learnables.Find(l => l.name.ToUpper().Replace(" ", "-") == predictedSign);
// If there is a feedback-object, we wil change its appearance
if (feedbackText != null && feedbackProgressImage != null)
{
Color col;
if (distCurrentSign < sign.thresholdDistance)
{
feedbackText.text = "Goed";
col = new Color(0x8b / 255.0f, 0xd4 / 255.0f, 0x5e / 255.0f);
}
else if (distCurrentSign < 1.5 * sign.thresholdDistance)
{
feedbackText.text = "Bijna";
col = new Color(0xf2 / 255.0f, 0x7f / 255.0f, 0x0c / 255.0f);
}
else if (distPredictSign < predSign.thresholdDistance)
{
feedbackText.text = $"Verkeerde gebaar: '{predSign.name}'";
col = new Color(0xf5 / 255.0f, 0x49 / 255.0f, 0x3d / 255.0f);
}
else
{
feedbackText.text = $"Detecteren ...";
col = new Color(0xf5 / 255.0f, 0x49 / 255.0f, 0x3d / 255.0f);
}
feedbackText.color = col;
feedbackProgressImage.color = col;
// Tween the feedback-bar
float oldValue = feedbackProgress.value;
float newValue = 1 - Mathf.Clamp(distCurrentSign - sign.thresholdDistance, 0.0f, 3.0f) / 3;
feedbackProgress.gameObject.Tween("FeedbackUpdate", oldValue, newValue, 0.2f, TweenScaleFunctions.CubicEaseInOut, (t) =>
{
if (feedbackProgress != null)
{
feedbackProgress.value = t.CurrentValue;
}
});
}
// The internal logic for the courses
if (distPredictSign < sign.thresholdDistance)
{
// Correct sign
if (predictedSign == currentSign)
{
yield return new WaitForSeconds(1.0f);
NextSignIfCorrect(currentSign, predictedSign);
timer = DateTime.Now;
previousIncorrectSign = null;
predictedSign = null;
}
// Incorrect sign
else
{
if (previousIncorrectSign != predictedSign)
{
timer = DateTime.Now;
previousIncorrectSign = predictedSign;
}
else if (predictedSign != null && currentSign != null &&
(DateTime.Now - timer).TotalSeconds > 2.0f)
{
NextSignIfCorrect(currentSign, predictedSign);
timer = DateTime.Now;
predictedSign = null;
previousIncorrectSign = null;
}
}
}
}
else if (feedbackProgress != null)
{
feedbackProgress.value = 0.0f;
}
}
}
///
/// Function to check equality between the current sign and the sign that the model predicted, if they are equal then the next sign is fetched.
///
///
public void NextSignIfCorrect(string current, string predicted)
{
if (!hasAnswered)
{
if (current == predicted)
{
hasAnswered = true;
var p = progress.learnables.Find((l) => l.name.ToUpper().Replace(" ", "-") == predicted);
progress.UpdateLearnable(p.name, 1.5f);
confettiAnimation.SetTrigger("Display Confetti");
StartCoroutine(WaitNextSign());
}
else
{
// currently ignore wrong signs as "J" doesn't work well enough
}
}
}
///
/// Wait 0.75 seconds and proceed to the next sign
///
///
private IEnumerator WaitNextSign()
{
// Wait for 0.75 seconds
yield return new WaitForSeconds(0.75f);
NextSign();
}
///
/// Function to check equality between the current sign and the sign that the model predicted, if they are equal then the next sign is fetched.
///
///
public void NextSignMultipleChoice(string current, string predicted)
{
if (!hasAnswered)
{
hasAnswered = true;
if (current == predicted)
{
progress.UpdateLearnable(predicted, 1.5f);
confettiAnimation.SetTrigger("Display Confetti");
}
else
{
progress.UpdateLearnable(predicted, -1.0f);
}
}
}
///
/// Callback for the 'back' button
///
public void ReturnToActivityScreen()
{
UserList.Save();
SystemController.GetInstance().BackToPreviousScene();
}
///
/// Returns panelId for testing
///
///
public int GetPanelId()
{
return panelId;
}
///
/// Returns currentSign for testing
///
///
public string GetCurrentSign()
{
Learnable sign = course.theme.learnables[currentWordIndex];
return sign.name.ToUpper().Replace(" ", "-");
}
///
/// Used for testing an out of bounds PanelId
///
///
public void CorruptPanelID()
{
corruptPanelId = true;
}
///
/// Needed to be able to test an out of bounds PanelId
///
///
public IEnumerator CallSetupPanel()
{
yield return SetupPanel();
}
///
/// Needed to be able to test an out of bounds PanelId
///
///
public void SetFeedbackProgress()
{
feedbackProgress.value = 0.0f;
}
///
/// Open multiple choice panel for testing
///
public IEnumerator SummonMultipleChoice()
{
CorruptPanelID();
CorruptedPanelIDValue = 1;
yield return CRNextSign();
}
}