551 lines
19 KiB
C#
551 lines
19 KiB
C#
using DigitalRuby.Tween;
|
|
using System;
|
|
using System.Collections;
|
|
using System.Linq;
|
|
using TMPro;
|
|
using UnityEngine;
|
|
using UnityEngine.UI;
|
|
using UnityEngine.Video;
|
|
|
|
/// <summary>
|
|
/// TemplateCourse scene manager
|
|
/// </summary>
|
|
public class CoursesController : AbstractFeedback
|
|
{
|
|
/// <summary>
|
|
/// Reference to the objet holding the title
|
|
/// </summary>
|
|
public TMP_Text courseTitle;
|
|
|
|
/// <summary>
|
|
/// The current user
|
|
/// </summary>
|
|
private User user;
|
|
|
|
/// <summary>
|
|
/// Current user progress for this course
|
|
/// </summary>
|
|
private PersistentDataController.SavedCourseProgress progress = null;
|
|
|
|
/// <summary>
|
|
/// ScriptableObject with list of all courses
|
|
/// </summary>
|
|
public CourseList courselist;
|
|
|
|
/// <summary>
|
|
/// Reference to Course ScriptableObject
|
|
/// </summary>
|
|
private Course course;
|
|
|
|
/// <summary>
|
|
/// Index of the current word/letter in the course.learnables list
|
|
/// </summary>
|
|
private int currentWordIndex = 0;
|
|
|
|
/// <summary>
|
|
/// This holds the amount of words in the course
|
|
/// </summary>
|
|
private int maxWords;
|
|
|
|
/// <summary>
|
|
/// The "finished" screen
|
|
/// </summary>
|
|
public GameObject ResultPanel;
|
|
|
|
/// <summary>
|
|
/// Reference to the title on the results panel
|
|
/// </summary>
|
|
public TMP_Text ResultsTitle;
|
|
|
|
/// <summary>
|
|
/// Reference to the description on the results panel
|
|
/// </summary>
|
|
public TMP_Text ResultsDecription;
|
|
|
|
/// <summary>
|
|
/// Button to go back to courses list
|
|
/// </summary>
|
|
public Button CoursesButton;
|
|
|
|
/// <summary>
|
|
/// DateTime containint the start moment
|
|
/// </summary>
|
|
private DateTime startMoment;
|
|
|
|
/// <summary>
|
|
/// Reference to the timeSpent UI
|
|
/// </summary>
|
|
public TMP_Text timeSpent;
|
|
|
|
/// <summary>
|
|
/// Reference to the feedback field
|
|
/// </summary>
|
|
private TMP_Text feedbackText;
|
|
|
|
/// <summary>
|
|
/// Reference to the progress bar
|
|
/// </summary>
|
|
private Slider feedbackProgress;
|
|
|
|
/// <summary>
|
|
/// Reference to the progress bar image, so we can add fancy colors
|
|
/// </summary>
|
|
private Image feedbackProgressImage;
|
|
|
|
public VideoPlayer videoPlayer;
|
|
|
|
/// <summary>
|
|
/// Timer to keep track of how long a incorrect sign is performed
|
|
/// </summary>
|
|
protected DateTime timer;
|
|
|
|
/// <summary>
|
|
/// Current predicted sign
|
|
/// </summary>
|
|
protected string predictedSign = null;
|
|
|
|
/// <summary>
|
|
/// Previous incorrect sign, so we can keep track whether the user is wrong or the user is still changing signs
|
|
/// </summary>
|
|
protected string previousIncorrectSign = null;
|
|
|
|
/// <summary>
|
|
/// Keeps track of what type of panel is currently being used
|
|
/// </summary>
|
|
protected int panelId = 0;
|
|
|
|
/// <summary>
|
|
/// Boolean used to check whether the user has already answered the question
|
|
/// </summary>
|
|
private bool hasAnswered = false;
|
|
|
|
/// <summary>
|
|
/// Boolean used to check whether SlideIn animation is playing
|
|
/// </summary>
|
|
private bool isNextSignInTransit = false;
|
|
|
|
/// <summary>
|
|
/// Reference to course progress bar
|
|
/// </summary>
|
|
public SlicedSlider progressBar;
|
|
|
|
/// <summary>
|
|
/// Reference to the animator of the confetti animation
|
|
/// </summary>
|
|
public Animator confettiAnimation;
|
|
|
|
/// <summary>
|
|
/// Panel with video&image prefab
|
|
/// </summary>
|
|
public GameObject panelSignWithVideoAndImagePrefab;
|
|
|
|
/// <summary>
|
|
/// Panel with image prefab
|
|
/// </summary>
|
|
public GameObject panelSignWithImagePrefab;
|
|
|
|
/// <summary>
|
|
/// Panel with multiplechoice prefab
|
|
/// </summary>
|
|
public GameObject panelMultipleChoicePrefab;
|
|
|
|
/// <summary>
|
|
/// Reference to the canvas to put the panels into
|
|
/// </summary>
|
|
public Transform canvas;
|
|
|
|
/// <summary>
|
|
/// Reference to the previous panel,
|
|
/// so it can be deleted when its done playing its exit animation
|
|
/// </summary>
|
|
private GameObject previousPanel = null;
|
|
|
|
/// <summary>
|
|
/// This function is called when the script is initialised.
|
|
/// It inactivatis the popup, finds a webcam to use and links it via the WebcamTexture to the display RawImage.
|
|
/// It takes the correct course from the courselist, using the courseIndex.
|
|
/// Then it checks whether or not the User has started the course yet, to possibly create a new progress atribute for the course.
|
|
/// Then it sets up the course-screen to display relevant information from the course-scriptable.
|
|
/// </summary>
|
|
void Start()
|
|
{
|
|
StartCourseController();
|
|
signPredictor.SetModel(course.theme.modelIndex);
|
|
AddSelfAsListener();
|
|
}
|
|
|
|
/// <summary>
|
|
/// Holds the course-specific logic to start the controller, it is seperated to allow the course to be reset (if that would become needed)
|
|
/// </summary>
|
|
public void StartCourseController()
|
|
{
|
|
// Setting up course
|
|
course = courselist.courses[courselist.currentCourseIndex];
|
|
maxWords = course.theme.learnables.Count;
|
|
|
|
// Reload from disk (course may be reset)
|
|
PersistentDataController.GetInstance().Load();
|
|
// Create entry in current user for keeping track of progress
|
|
user = UserList.GetCurrentUser();
|
|
progress = user.GetCourseProgress(course.index);
|
|
if (progress == null)
|
|
{
|
|
progress = new PersistentDataController.SavedCourseProgress();
|
|
progress.courseIndex = course.index;
|
|
int index = 0;
|
|
foreach (Learnable learnable in course.theme.learnables)
|
|
{
|
|
progress.AddLearnable(learnable.name, index++);
|
|
}
|
|
user.AddCourseProgress(progress);
|
|
}
|
|
UserList.Save();
|
|
progressBar.fillAmount = progress.progress;
|
|
|
|
currentWordIndex = 0;
|
|
previousPanel = SetupPanel();
|
|
|
|
// Hide the result panel
|
|
ResultPanel.SetActive(false);
|
|
// Set the startTime
|
|
startMoment = DateTime.Now;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Fetch the next sign and its panel type
|
|
/// </summary>
|
|
/// <returns>A tuple of {next sign index, panel type}</returns>
|
|
/// <remarks>
|
|
/// The different panel types:<br></br>
|
|
/// 0 : panelSignWithVideoAndImagePrefab<br></br>
|
|
/// 1 : panelMultipleChoicePrefab<br></br>
|
|
/// 2 : panelSignWithImagePrefab
|
|
/// </remarks>
|
|
private Tuple<int, int> FetchSign()
|
|
{
|
|
PersistentDataController.SavedLearnableProgress learnable = progress.GetRandomLearnable();
|
|
int panelChosen;
|
|
if (course.theme.modelIndex == ModelIndex.NONE)
|
|
{
|
|
// only multiple choice works in preview mode
|
|
panelChosen = 1;
|
|
}
|
|
else if (learnable.progress > 2.0f)
|
|
{
|
|
panelChosen = 2;
|
|
}
|
|
else if (learnable.progress > 1.0f)
|
|
{
|
|
panelChosen = 1;
|
|
}
|
|
else
|
|
{
|
|
panelChosen = 0;
|
|
}
|
|
return Tuple.Create(learnable.index, panelChosen);
|
|
}
|
|
|
|
/// <summary>
|
|
/// This function is called when the next-sign button is pressed.
|
|
/// It increased the wordindex and fetches new videos/images if index<max, because then the coure is not fincished yet.
|
|
/// If the maximum is reached, finishcourse is called to save the "finished" progress to the user.
|
|
/// </summary>
|
|
public void NextSign()
|
|
{
|
|
// This function is also called (async) when pressing the 'Gebaar overslaan' button,
|
|
// so check for condition so we don't skip multiple signs
|
|
if (isNextSignInTransit || maxWords <= progress.completedLearnables)
|
|
return;
|
|
|
|
progress.progress = (float)progress.completedLearnables / (float)maxWords;
|
|
progressBar.fillAmount = progress.progress;
|
|
|
|
// Update UI if course is not finished yet
|
|
if (progress.completedLearnables < maxWords)
|
|
{
|
|
// Set next sign/video/image
|
|
StartCoroutine(CRNextSign());
|
|
}
|
|
// Finish course and record progress
|
|
if (progress.completedLearnables == maxWords)
|
|
{
|
|
FinishCourse();
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Coroutine for going to the next sign
|
|
/// </summary>
|
|
/// <returns></returns>
|
|
private IEnumerator CRNextSign()
|
|
{
|
|
isNextSignInTransit = true;
|
|
GameObject newPanel = SetupPanel();
|
|
previousPanel.transform.SetAsFirstSibling();
|
|
newPanel.GetComponent<Animator>().SetTrigger("Slide Panel In");
|
|
if (previousPanel != null)
|
|
previousPanel.GetComponent<Animator>().SetTrigger("Slide Panel Out");
|
|
|
|
yield return new WaitForSeconds(1.0f);
|
|
|
|
confettiAnimation.ResetTrigger("Display Confetti");
|
|
GameObject.Destroy(previousPanel);
|
|
previousPanel = newPanel;
|
|
hasAnswered = false;
|
|
isNextSignInTransit = false;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Setup a new panel
|
|
/// </summary>
|
|
/// <returns>Reference to the GameObject of the panel</returns>
|
|
private GameObject SetupPanel()
|
|
{
|
|
(currentWordIndex, panelId) = FetchSign().ToValueTuple();
|
|
switch (panelId)
|
|
{
|
|
case 0:
|
|
{
|
|
GameObject panel = GameObject.Instantiate(panelSignWithVideoAndImagePrefab, canvas);
|
|
panel.transform.SetAsFirstSibling();
|
|
|
|
PanelWithVideoAndImage script = panel.GetComponent<PanelWithVideoAndImage>();
|
|
script.signs = course.theme.learnables;
|
|
script.currentSignIndex = currentWordIndex;
|
|
script.isPreview = (course.theme.modelIndex == ModelIndex.NONE);
|
|
script.videoPlayer = videoPlayer;
|
|
feedbackProgress = script.feedbackProgressBar;
|
|
feedbackProgressImage = script.feedbackProgressImage;
|
|
feedbackText = script.feedbackText;
|
|
script.Display();
|
|
signPredictor.SwapScreen(script.webcamScreen);
|
|
courseTitle.text = "Voer het gebaar uit voor \"" + course.theme.learnables[currentWordIndex].name + "\"";
|
|
return panel;
|
|
}
|
|
case 1:
|
|
{
|
|
GameObject panel = GameObject.Instantiate(panelMultipleChoicePrefab, canvas);
|
|
panel.transform.SetAsFirstSibling();
|
|
|
|
PanelMultipleChoice script = panel.GetComponent<PanelMultipleChoice>();
|
|
script.signs = course.theme.learnables;
|
|
script.currentSignIndex = currentWordIndex;
|
|
script.videoPlayer = videoPlayer;
|
|
script.courseController = this;
|
|
script.progress = progress;
|
|
script.isFingerSpelling = course.theme.title == "Handalfabet";
|
|
script.Display();
|
|
signPredictor.SwapScreen(script.webcamScreen);
|
|
courseTitle.text = "Welk gebaar zie je hier?";
|
|
return panel;
|
|
}
|
|
case 2:
|
|
{
|
|
GameObject panel = GameObject.Instantiate(panelSignWithImagePrefab, canvas);
|
|
panel.transform.SetAsFirstSibling();
|
|
|
|
PanelWithImage script = panel.GetComponent<PanelWithImage>();
|
|
script.signs = course.theme.learnables;
|
|
script.currentSignIndex = currentWordIndex;
|
|
script.isPreview = (course.theme.modelIndex == ModelIndex.NONE);
|
|
feedbackProgress = script.feedbackProgressBar;
|
|
feedbackProgressImage = script.feedbackProgressImage;
|
|
feedbackText = script.feedbackText;
|
|
script.Display();
|
|
signPredictor.SwapScreen(script.webcamScreen);
|
|
courseTitle.text = "Voer het gebaar uit voor \"" + course.theme.learnables[currentWordIndex].name + "\"";
|
|
return panel;
|
|
}
|
|
}
|
|
return null;
|
|
}
|
|
|
|
/// <summary>
|
|
/// finishcourse is called to save the "finished" progress to the user.
|
|
/// </summary>
|
|
public void FinishCourse()
|
|
{
|
|
// Show the "finished" screen
|
|
ResultPanel.SetActive(true);
|
|
|
|
// Set the correct title
|
|
ResultsTitle.text = course.title + " voltooid!";
|
|
|
|
// Set the correct description
|
|
ResultsDecription.text = "Goed gedaan! Je kan nu spelletjes spelen met " + course.title + " om verder te oefenen!";
|
|
|
|
// Set the total time spent UI
|
|
TimeSpan time = DateTime.Now - startMoment;
|
|
timeSpent.text = time.ToString(@"hh\:mm\:ss");
|
|
|
|
// Link button
|
|
CoursesButton.onClick.AddListener(() => { SystemController.GetInstance().BackToPreviousScene(); });
|
|
|
|
progress.progress = 1.0f;
|
|
UserList.Save();
|
|
}
|
|
|
|
/// <summary>
|
|
/// The updateFunction that is called when new probabilities become available
|
|
/// </summary>
|
|
/// <returns></returns>
|
|
protected override IEnumerator UpdateFeedback()
|
|
{
|
|
// Check if the current word index is still in bounds, and if the current panel type is not multiple choice
|
|
if (currentWordIndex < course.theme.learnables.Count && panelId != 1 && !hasAnswered)
|
|
{
|
|
// Get current sign
|
|
Learnable sign = course.theme.learnables[currentWordIndex];
|
|
string currentSign = sign.name.ToUpper().Replace(" ", "-");
|
|
|
|
// Get the predicted sign
|
|
if (signPredictor != null && signPredictor.learnableProbabilities != null &&
|
|
currentSign != null && signPredictor.learnableProbabilities.ContainsKey(currentSign))
|
|
{
|
|
Debug.Log($"{signPredictor.learnableProbabilities.Aggregate("", (t, e) => $"{t}{e.Key}={e.Value}, ")}");
|
|
float accCurrentSign = signPredictor.learnableProbabilities[currentSign];
|
|
|
|
// Get highest predicted sign
|
|
string predictedSign = signPredictor.learnableProbabilities.Aggregate((a, b) => a.Value > b.Value ? a : b).Key;
|
|
float accPredictSign = signPredictor.learnableProbabilities[predictedSign];
|
|
Learnable predSign = course.theme.learnables.Find(l => l.name.ToUpper().Replace(" ", "-") == predictedSign);
|
|
|
|
if (feedbackText != null && feedbackProgressImage != null)
|
|
{
|
|
Color col;
|
|
if (accCurrentSign > sign.thresholdPercentage)
|
|
{
|
|
feedbackText.text = "Goed";
|
|
col = new Color(0x8b / 255.0f, 0xd4 / 255.0f, 0x5e / 255.0f);
|
|
}
|
|
else if (accCurrentSign > 0.9 * sign.thresholdPercentage)
|
|
{
|
|
feedbackText.text = "Bijna";
|
|
col = new Color(0xf2 / 255.0f, 0x7f / 255.0f, 0x0c / 255.0f);
|
|
}
|
|
else if (accPredictSign > predSign.thresholdPercentage)
|
|
{
|
|
feedbackText.text = $"Verkeerde gebaar: '{predSign.name}'";
|
|
col = new Color(0xf5 / 255.0f, 0x49 / 255.0f, 0x3d / 255.0f);
|
|
accCurrentSign = 0.0f;
|
|
}
|
|
else
|
|
{
|
|
feedbackText.text = $"Detecteren ...";
|
|
col = new Color(0xf5 / 255.0f, 0x49 / 255.0f, 0x3d / 255.0f);
|
|
//accCurrentSign = 0.0f;
|
|
}
|
|
|
|
feedbackText.color = col;
|
|
feedbackProgressImage.color = col;
|
|
|
|
float oldValue = feedbackProgress.value;
|
|
// use an exponential scale
|
|
float newValue = Mathf.Exp(4 * (Mathf.Clamp(accCurrentSign / sign.thresholdPercentage, 0.0f, 1.0f) - 1.0f));
|
|
feedbackProgress.gameObject.Tween("FeedbackUpdate", oldValue, newValue, 0.2f, TweenScaleFunctions.CubicEaseInOut, (t) =>
|
|
{
|
|
if (feedbackProgress != null)
|
|
{
|
|
feedbackProgress.value = t.CurrentValue;
|
|
}
|
|
});
|
|
}
|
|
|
|
if (accPredictSign > sign.thresholdPercentage)
|
|
{
|
|
// Correct sign
|
|
if (predictedSign == currentSign)
|
|
{
|
|
yield return new WaitForSeconds(1.0f);
|
|
NextSignIfCorrect(currentSign, predictedSign);
|
|
timer = DateTime.Now;
|
|
previousIncorrectSign = null;
|
|
predictedSign = null;
|
|
}
|
|
// Incorrect sign
|
|
else
|
|
{
|
|
if (previousIncorrectSign != predictedSign)
|
|
{
|
|
timer = DateTime.Now;
|
|
previousIncorrectSign = predictedSign;
|
|
}
|
|
else if (predictedSign != null && currentSign != null &&
|
|
(DateTime.Now - timer).TotalSeconds > 2.0f)
|
|
{
|
|
NextSignIfCorrect(currentSign, predictedSign);
|
|
timer = DateTime.Now;
|
|
predictedSign = null;
|
|
previousIncorrectSign = null;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
else if (feedbackProgress != null)
|
|
{
|
|
feedbackProgress.value = 0.0f;
|
|
}
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Function to check equality between the current sign and the sign that the model predicted, if they are equal then the next sign is fetched.
|
|
/// </summary>
|
|
/// <param name="predicted"></param>
|
|
public void NextSignIfCorrect(string current, string predicted)
|
|
{
|
|
if (!hasAnswered)
|
|
{
|
|
if (current == predicted)
|
|
{
|
|
hasAnswered = true;
|
|
progress.UpdateLearnable(predicted, 1.5f);
|
|
confettiAnimation.SetTrigger("Display Confetti");
|
|
StartCoroutine(WaitNextSign());
|
|
}
|
|
else
|
|
{
|
|
// currently ignore wrong signs as "J" doesn't work well enough
|
|
}
|
|
}
|
|
|
|
}
|
|
|
|
private IEnumerator WaitNextSign()
|
|
{
|
|
// Wait for 0.75 seconds
|
|
yield return new WaitForSeconds(0.75f);
|
|
NextSign();
|
|
}
|
|
|
|
/// <summary>
|
|
/// Function to check equality between the current sign and the sign that the model predicted, if they are equal then the next sign is fetched.
|
|
/// </summary>
|
|
/// <param name="predicted"></param>
|
|
public void NextSignMultipleChoice(string current, string predicted)
|
|
{
|
|
if (!hasAnswered)
|
|
{
|
|
hasAnswered = true;
|
|
if (current == predicted)
|
|
{
|
|
progress.UpdateLearnable(predicted, 1.5f);
|
|
confettiAnimation.SetTrigger("Display Confetti");
|
|
}
|
|
else
|
|
{
|
|
progress.UpdateLearnable(predicted, -1.0f);
|
|
}
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Callback for the 'back' button
|
|
/// </summary>
|
|
public void ReturnToActivityScreen()
|
|
{
|
|
UserList.Save();
|
|
SystemController.GetInstance().BackToPreviousScene();
|
|
}
|
|
}
|