using DigitalRuby.Tween; using System; using System.Collections; using System.Collections.Generic; using System.Linq; using TMPro; using UnityEngine; using UnityEngine.UI; using UnityEngine.Video; /// /// TemplateCourse scene manager /// public class CoursesController : AbstractFeedback { /// /// Reference to the objet holding the title /// public TMP_Text courseTitle; /// /// The current user /// private User user; /// /// Current user progress for this course /// private PersistentDataController.SavedCourseProgress progress = null; /// /// ScriptableObject with list of all courses /// public CourseList courselist; /// /// Reference to Course ScriptableObject /// private Course course; /// /// Index of the current word/letter in the course.learnables list /// private int currentWordIndex = 0; /// /// This holds the amount of words in the course /// private int maxWords; /// /// The "finished" screen /// public GameObject ResultPanel; /// /// Reference to the title on the results panel /// public TMP_Text ResultsTitle; /// /// Reference to the description on the results panel /// public TMP_Text ResultsDecription; /// /// Button to go back to courses list /// public Button CoursesButton; /// /// DateTime containint the start moment /// private DateTime startMoment; /// /// Reference to the timeSpent UI /// public TMP_Text timeSpent; /// /// Reference to the feedback field /// private TMP_Text feedbackText; /// /// Reference to the progress bar /// private Slider feedbackProgress; /// /// Reference to the progress bar image, so we can add fancy colors /// private Image feedbackProgressImage; /// /// Reference to the video player /// public VideoPlayer videoPlayer; /// /// Timer to keep track of how long a incorrect sign is performed /// protected DateTime timer; /// /// Current predicted sign /// protected string predictedSign = null; /// /// Previous incorrect sign, so we can keep track whether the user is wrong or the user is still changing signs /// protected string previousIncorrectSign = null; /// /// Keeps track of what type of panel is currently being used /// protected int panelId = 0; /// /// Boolean used to check whether the user has already answered the question /// private bool hasAnswered = false; /// /// Boolean used to check whether SlideIn animation is playing /// private bool isNextSignInTransit = false; /// /// Reference to course progress bar /// public SlicedSlider progressBar; /// /// Reference to the animator of the confetti animation /// public Animator confettiAnimation; /// /// Panel with video&image prefab /// public GameObject panelSignWithVideoAndImagePrefab; /// /// Panel with image prefab /// public GameObject panelSignWithImagePrefab; /// /// Panel with multiplechoice prefab /// public GameObject panelMultipleChoicePrefab; /// /// Reference to the canvas to put the panels into /// public Transform canvas; /// /// Reference to the previous panel, /// so it can be deleted when its done playing its exit animation /// private GameObject previousPanel = null; /// /// Boolean used to make a test possible /// private bool corruptPanelId = false; /// /// Corrupted PanelID Value /// private int CorruptedPanelIDValue = 999; /// /// This function is called when the script is initialised. /// It inactivatis the popup, finds a webcam to use and links it via the WebcamTexture to the display RawImage. /// It takes the correct course from the courselist, using the courseIndex. /// Then it checks whether or not the User has started the course yet, to possibly create a new progress atribute for the course. /// Then it sets up the course-screen to display relevant information from the course-scriptable. /// void Start() { StartCourseController(); signPredictor.SetSignsList(GetSignsList()); signPredictor.SetModel(course.theme.modelIndex); AddSelfAsListener(); } /// /// Fetches all the strings of the signs of the course /// /// The signsList that needs to be passed to the signPredictor private List GetSignsList() { List signsList = new List(); foreach (Learnable learnable in course.theme.learnables) { signsList.Add(learnable.name); } return signsList; } /// /// Holds the course-specific logic to start the controller, it is seperated to allow the course to be reset (if that would become needed) /// public void StartCourseController() { // Setting up course course = courselist.courses[courselist.currentCourseIndex]; maxWords = course.theme.learnables.Count; // Reload from disk (course may be reset) PersistentDataController.GetInstance().Load(); // Create entry in current user for keeping track of progress user = UserList.GetCurrentUser(); progress = user.GetCourseProgress(course.index); if (progress == null) { progress = new PersistentDataController.SavedCourseProgress(); progress.courseIndex = course.index; int index = 0; foreach (Learnable learnable in course.theme.learnables) { progress.AddLearnable(learnable.name, index++); } user.AddCourseProgress(progress); } UserList.Save(); progressBar.fillAmount = progress.progress; currentWordIndex = 0; previousPanel = SetupPanel(); // Hide the result panel ResultPanel.SetActive(false); // Set the startTime startMoment = DateTime.Now; } /// /// Fetch the next sign and its panel type /// /// A tuple of {next sign index, panel type} /// /// The different panel types:

/// 0 : panelSignWithVideoAndImagePrefab

/// 1 : panelMultipleChoicePrefab

/// 2 : panelSignWithImagePrefab ///
private Tuple FetchSign() { PersistentDataController.SavedLearnableProgress learnable = progress.GetRandomLearnable(); int panelChosen; if (learnable.progress > 2.0f) { panelChosen = 2; } else if (learnable.progress > 1.0f) { panelChosen = 1; } else { panelChosen = 0; } return Tuple.Create(learnable.index, panelChosen); } /// /// This function is called when the next-sign button is pressed. /// It increased the wordindex and fetches new videos/images if index public void NextSign() { // This function is also called (async) when pressing the 'Gebaar overslaan' button, // so check for condition so we don't skip multiple signs //if (isNextSignInTransit || maxWords < progress.completedLearnables) if (isNextSignInTransit) return; // Code for preview-progress, skipping should give progress unless it is multipleChoice if (course.theme.modelIndex == ModelIndex.NONE) { string currentName = course.theme.learnables[currentWordIndex].name; // This works both to allow panel 0 to allow progress via skipping and also to allow panel 2 to be skipped. if (progress.FindLearnable(currentName).progress <= 1f || progress.FindLearnable(currentName).progress >= 2f) progress.UpdateLearnable(currentName, 1.5f); } progress.progress = (float)progress.completedLearnables / (float)maxWords; progressBar.fillAmount = progress.progress; // Update UI if course is not finished yet if (progress.completedLearnables < maxWords) { // Set next sign/video/image StartCoroutine(CRNextSign()); } // Finish course and record progress else { FinishCourse(); } } /// /// Coroutine for going to the next sign /// /// private IEnumerator CRNextSign() { isNextSignInTransit = true; GameObject newPanel = SetupPanel(); previousPanel.transform.SetAsFirstSibling(); newPanel.GetComponent().SetTrigger("Slide Panel In"); if (previousPanel != null) previousPanel.GetComponent().SetTrigger("Slide Panel Out"); yield return new WaitForSeconds(1.0f); confettiAnimation.ResetTrigger("Display Confetti"); GameObject.Destroy(previousPanel); previousPanel = newPanel; hasAnswered = false; isNextSignInTransit = false; } /// /// Setup a new panel /// /// Reference to the GameObject of the panel private GameObject SetupPanel() { if (corruptPanelId == true) { (currentWordIndex, panelId) = (1, CorruptedPanelIDValue); } else { (currentWordIndex, panelId) = FetchSign().ToValueTuple(); } switch (panelId) { case 0: { GameObject panel = GameObject.Instantiate(panelSignWithVideoAndImagePrefab, canvas); panel.transform.SetAsFirstSibling(); PanelWithVideoAndImage script = panel.GetComponent(); script.signs = course.theme.learnables; script.currentSignIndex = currentWordIndex; script.isPreview = (course.theme.modelIndex == ModelIndex.NONE); script.videoPlayer = videoPlayer; feedbackProgress = script.feedbackProgressBar; feedbackProgressImage = script.feedbackProgressImage; feedbackText = script.feedbackText; script.Display(); signPredictor.SwapScreen(script.webcamScreen); courseTitle.text = "Voer het gebaar uit voor \"" + course.theme.learnables[currentWordIndex].name + "\""; return panel; } case 1: { GameObject panel = GameObject.Instantiate(panelMultipleChoicePrefab, canvas); panel.transform.SetAsFirstSibling(); PanelMultipleChoice script = panel.GetComponent(); script.signs = course.theme.learnables; script.currentSignIndex = currentWordIndex; script.videoPlayer = videoPlayer; script.courseController = this; script.progress = progress; script.isFingerSpelling = course.theme.title == "Handalfabet"; script.Display(); signPredictor.SwapScreen(script.webcamScreen); courseTitle.text = "Welk gebaar zie je hier?"; return panel; } case 2: { GameObject panel = GameObject.Instantiate(panelSignWithImagePrefab, canvas); panel.transform.SetAsFirstSibling(); PanelWithImage script = panel.GetComponent(); script.signs = course.theme.learnables; script.currentSignIndex = currentWordIndex; script.isPreview = (course.theme.modelIndex == ModelIndex.NONE); feedbackProgress = script.feedbackProgressBar; feedbackProgressImage = script.feedbackProgressImage; feedbackText = script.feedbackText; script.Display(); signPredictor.SwapScreen(script.webcamScreen); courseTitle.text = "Voer het gebaar uit voor \"" + course.theme.learnables[currentWordIndex].name + "\""; return panel; } } return null; } /// /// finishcourse is called to save the "finished" progress to the user. /// public void FinishCourse() { // Show the "finished" screen ResultPanel.SetActive(true); // Set the correct title ResultsTitle.text = course.title + " is voltooid!"; // Set the correct description ResultsDecription.text = "Goed gedaan! Je kan nu spelletjes spelen met " + course.title + " om verder te oefenen!"; // Set the total time spent UI TimeSpan time = DateTime.Now - startMoment; timeSpent.text = time.ToString(@"hh\:mm\:ss"); // Link button CoursesButton.onClick.AddListener(() => { SystemController.GetInstance().BackToPreviousScene(); }); progress.progress = 1.0f; UserList.Save(); } /// /// The updateFunction that is called when new probabilities become available /// /// protected override IEnumerator UpdateFeedback() { // Check if the current word index is still in bounds, and if the current panel type is not multiple choice if (currentWordIndex < course.theme.learnables.Count && panelId != 1 && !hasAnswered) { // Get current sign Learnable sign = course.theme.learnables[currentWordIndex]; string currentSign = sign.name.ToUpper().Replace(" ", "-"); // Get the predicted sign if (signPredictor != null && signPredictor.learnableProbabilities != null && currentSign != null && signPredictor.learnableProbabilities.ContainsKey(currentSign)) { //Debug.Log($"{signPredictor.learnableProbabilities.Aggregate("", (t, e) => $"{t}{e.Key}={e.Value}, ")}"); float distCurrentSign = signPredictor.learnableProbabilities[currentSign]; // Get highest predicted sign string predictedSign = signPredictor.learnableProbabilities.Aggregate((a, b) => a.Value < b.Value ? a : b).Key; float distPredictSign = signPredictor.learnableProbabilities[predictedSign]; Learnable predSign = course.theme.learnables.Find(l => l.name.ToUpper().Replace(" ", "-") == predictedSign); // If there is a feedback-object, we wil change its appearance if (feedbackText != null && feedbackProgressImage != null) { Color col; if (distCurrentSign < sign.thresholdDistance) { feedbackText.text = "Goed"; col = new Color(0x8b / 255.0f, 0xd4 / 255.0f, 0x5e / 255.0f); } else if (distCurrentSign < 1.5 * sign.thresholdDistance) { feedbackText.text = "Bijna"; col = new Color(0xf2 / 255.0f, 0x7f / 255.0f, 0x0c / 255.0f); } else if (distPredictSign < predSign.thresholdDistance) { feedbackText.text = $"Verkeerde gebaar: '{predSign.name}'"; col = new Color(0xf5 / 255.0f, 0x49 / 255.0f, 0x3d / 255.0f); } else { feedbackText.text = $"Detecteren ..."; col = new Color(0xf5 / 255.0f, 0x49 / 255.0f, 0x3d / 255.0f); } feedbackText.color = col; feedbackProgressImage.color = col; // Tween the feedback-bar float oldValue = feedbackProgress.value; float newValue = 1 - Mathf.Clamp(distCurrentSign - sign.thresholdDistance, 0.0f, 3.0f) / 3; feedbackProgress.gameObject.Tween("FeedbackUpdate", oldValue, newValue, 0.2f, TweenScaleFunctions.CubicEaseInOut, (t) => { if (feedbackProgress != null) { feedbackProgress.value = t.CurrentValue; } }); } // The internal logic for the courses if (distPredictSign < sign.thresholdDistance) { // Correct sign if (predictedSign == currentSign) { yield return new WaitForSeconds(1.0f); NextSignIfCorrect(currentSign, predictedSign); timer = DateTime.Now; previousIncorrectSign = null; predictedSign = null; } // Incorrect sign else { if (previousIncorrectSign != predictedSign) { timer = DateTime.Now; previousIncorrectSign = predictedSign; } else if (predictedSign != null && currentSign != null && (DateTime.Now - timer).TotalSeconds > 2.0f) { NextSignIfCorrect(currentSign, predictedSign); timer = DateTime.Now; predictedSign = null; previousIncorrectSign = null; } } } } else if (feedbackProgress != null) { feedbackProgress.value = 0.0f; } } } /// /// Function to check equality between the current sign and the sign that the model predicted, if they are equal then the next sign is fetched. /// /// public void NextSignIfCorrect(string current, string predicted) { if (!hasAnswered) { if (current == predicted) { hasAnswered = true; var p = progress.learnables.Find((l) => l.name.ToUpper().Replace(" ", "-") == predicted); progress.UpdateLearnable(p.name, 1.5f); confettiAnimation.SetTrigger("Display Confetti"); StartCoroutine(WaitNextSign()); } else { // currently ignore wrong signs as "J" doesn't work well enough } } } /// /// Wait 0.75 seconds and proceed to the next sign /// /// private IEnumerator WaitNextSign() { // Wait for 0.75 seconds yield return new WaitForSeconds(0.75f); NextSign(); } /// /// Function to check equality between the current sign and the sign that the model predicted, if they are equal then the next sign is fetched. /// /// public void NextSignMultipleChoice(string current, string predicted) { if (!hasAnswered) { hasAnswered = true; if (current == predicted) { progress.UpdateLearnable(predicted, 1.5f); confettiAnimation.SetTrigger("Display Confetti"); } else { progress.UpdateLearnable(predicted, -1.0f); } } } /// /// Callback for the 'back' button /// public void ReturnToActivityScreen() { UserList.Save(); SystemController.GetInstance().BackToPreviousScene(); } /// /// Returns panelId for testing /// /// public int GetPanelId() { return panelId; } /// /// Returns currentSign for testing /// /// public string GetCurrentSign() { Learnable sign = course.theme.learnables[currentWordIndex]; return sign.name.ToUpper().Replace(" ", "-"); } /// /// Used for testing an out of bounds PanelId /// /// public void CorruptPanelID() { corruptPanelId = true; } /// /// Needed to be able to test an out of bounds PanelId /// /// public IEnumerator CallSetupPanel() { yield return SetupPanel(); } /// /// Needed to be able to test an out of bounds PanelId /// /// public void SetFeedbackProgress() { feedbackProgress.value = 0.0f; } /// /// Open multiple choice panel for testing /// public IEnumerator SummonMultipleChoice() { CorruptPanelID(); CorruptedPanelIDValue = 1; yield return CRNextSign(); } }