using DigitalRuby.Tween; using System; using System.Collections; using TMPro; using UnityEngine; using UnityEngine.UI; using UnityEngine.Video; /// /// TemplateCourse scene manager /// public class CoursesController : AbstractFeedback { public GameObject feedbackProgressBar; public GameObject previewMessage; /// /// Reference to instructional video player /// public VideoPlayer player; /// /// Reference to pause button /// public Button button; /// /// Reference to sprite for the pause button /// public Sprite pauseSprite; /// /// Reference to the image for displaying the current words sprite /// public Image wordImage; /// /// Reference to the text object for displaying the current word /// public TMP_Text title; /// /// The current user /// private User user; /// /// Current user progress for this course /// private PersistentDataController.SavedCourseProgress progress = null; /// /// ScriptableObject with list of all courses /// public CourseList courselist; /// /// Reference to Course ScriptableObject /// private Course course; /// /// Index of the current word/letter in the course.learnables list /// private int currentWordIndex = 0; /// /// This holds the amount of words in the course /// private int maxWords; /// /// Number of correct words so far /// (can be modified to a list or something like that to give better feedback) /// private int correctWords = 0; /// /// The "finished" screen /// public GameObject ResultPanel; /// /// Reference to the title on the results panel /// public TMP_Text ResultsTitle; /// /// Reference to the description on the results panel /// public TMP_Text ResultsDecription; /// /// Button to go back to courses list /// public Button CoursesButton; /// /// DateTime containint the start moment /// private DateTime startMoment; /// /// Reference to the timeSpent UI /// public TMP_Text timeSpent; /// /// Reference to the feedback field /// public TMP_Text feedbackText; /// /// Reference to the progress bar /// public Slider feedbackProgress; /// /// Reference to the progress bar image, so we can add fancy colors /// public Image feedbackProgressImage; /// /// Timer to keep track of how long a incorrect sign is performed /// protected DateTime timer; /// /// Current predicted sign /// protected string predictedSign = null; /// /// Previous incorrect sign, so we can keep track whether the user is wrong or the user is still changing signs /// protected string previousIncorrectSign = null; /// /// This function is called when the script is initialised. /// It inactivatis the popup, finds a webcam to use and links it via the WebcamTexture to the display RawImage. /// It takes the correct course from the courselist, using the courseIndex. /// Then it checks whether or not the User has started the course yet, to possibly create a new progress atribute for the course. /// Then it sets up the course-screen to display relevant information from the course-scriptable. /// void Start() { StartCourseController(); signPredictor.ChangeModel(course.theme.modelIndex); AddSelfAsListener(); } /// /// Holds the course-specific logic to start the controller, it is seperated to allow the course to be reset (if that would become needed) /// public void StartCourseController() { // Setting up course course = courselist.courses[courselist.currentCourseIndex]; maxWords = course.theme.learnables.Count; // Show preview messages if there is no model feedbackProgressBar.SetActive(course.theme.modelIndex != ModelIndex.NONE); previewMessage.SetActive(course.theme.modelIndex == ModelIndex.NONE); // Create entry in current user for keeping track of progress //PersistentDataController pdc = PersistentDataController.GetInstance(); //pdc.Load(); user = UserList.GetCurrentUser(); progress = user.GetCourseProgress(course.index); if (progress == null) { progress = new PersistentDataController.SavedCourseProgress(); progress.courseIndex = course.index; user.AddCourseProgress(progress); } UserList.Save(); // Force the videoplayer to add bars to preserve aspect ratio player.aspectRatio = VideoAspectRatio.FitInside; // Setup UI button.image.sprite = pauseSprite; title.text = course.title; NextVideo(); NextImage(); // Hide the result panel ResultPanel.SetActive(false); // Set the startTime startMoment = DateTime.Now; } /// /// This function uses the word_i integer to grab the correct video from the course.learnabels. /// When it has this video, it will load it into the videoplayer and set it to start. /// private void NextVideo() { player.clip = course.theme.learnables[currentWordIndex].clip; // This loads first frame, so that it can be used as a sort-of preview for the video player.Play(); // As the video will start playiing -> hide button Color col = button.image.color; col.a = 0; button.image.color = col; } /// /// This function uses the word_i integer to grab the correct image from the course.learnabels. /// Then it simply loads it into wordImage so that it can be displayed. /// private void NextImage() { wordImage.sprite = course.theme.learnables[currentWordIndex].image; } /// /// This function is called when the pause-button is pressed on the video. /// It switches between playing and pausing the video. /// It then makes the button invisible when the video is playing, or visible when it's paused. /// public void Pause() { if (!player.isPlaying) { // Play video and hide button player.Play(); Color col = button.image.color; col.a = 0; button.image.color = col; } else { // Pause video and show button player.Pause(); Color col = button.image.color; col.a = 255; button.image.color = col; } } /// /// This function is called when the next-sign button is pressed. /// It increased the wordindex and fetches new videos/images if index public void NextSign() { // If the currentindex >= maxwords, it indicated that the course is already finished, running the next code is the meaningless. if (currentWordIndex >= maxWords) { return; } // Goto the next word/letter currentWordIndex++; // TODO: fix correct word count correctWords++; progress.progress = (float)correctWords / (float)maxWords; UserList.Save(); // Update UI if course is not finished yet if (currentWordIndex < maxWords) { NextVideo(); NextImage(); } // Finish course and record progress else { FinishCourse(); } } /// /// finishcourse is called to save the "finished" progress to the user. /// public void FinishCourse() { // Show the "finished" screen ResultPanel.SetActive(true); // Set the correct title ResultsTitle.text = course.title + " voltooid!"; // Set the correct description ResultsDecription.text = "Goed gedaan! Je kan nu spelletjes spelen met " + course.title + " om verder te oefenen!"; // Set the total time spent UI TimeSpan time = DateTime.Now - startMoment; timeSpent.text = time.ToString(@"hh\:mm\:ss"); // Link button CoursesButton.onClick.AddListener(() => { SystemController.GetInstance().BackToPreviousScene(); }); progress.progress = 1.0f; UserList.Save(); } /// /// The updateFunction that is called when new probabilities become available /// /// protected override IEnumerator UpdateFeedback() { // Get current sign string currentSign = course.theme.learnables[currentWordIndex].name; // Get the predicted sign if (signPredictor != null && signPredictor.learnableProbabilities != null && currentSign != null && signPredictor.learnableProbabilities.ContainsKey(currentSign)) { float accuracy = signPredictor.learnableProbabilities[currentSign]; if (feedbackText != null && feedbackProgressImage != null) { if (accuracy > 0.90) { feedbackText.text = "Goed"; feedbackText.color = Color.green; feedbackProgressImage.color = Color.green; } else if (accuracy > 0.80) { feedbackText.text = "Bijna..."; Color col = new Color(0xff / 255.0f, 0x66 / 255.0f, 0x00 / 255.0f); feedbackText.color = col; feedbackProgressImage.color = col; } else { feedbackText.text = "Detecteren..."; feedbackText.color = Color.red; feedbackProgressImage.color = Color.red; } float oldValue = feedbackProgress.value; // use an exponential scale float newValue = Mathf.Exp(4 * (accuracy - 1.0f)); feedbackProgress.gameObject.Tween("FeedbackUpdate", oldValue, newValue, 0.2f, TweenScaleFunctions.CubicEaseInOut, (t) => { if (feedbackProgress != null) { feedbackProgress.value = t.CurrentValue; } }); } // Check whether (in)correct sign has high accuracy foreach (var kv in signPredictor.learnableProbabilities) { if (kv.Value > 0.90) { predictedSign = kv.Key; // Correct sign if (predictedSign == currentSign) { yield return new WaitForSeconds(1.0f); CheckEquality(predictedSign); timer = DateTime.Now; predictedSign = null; previousIncorrectSign = null; } // Incorrect sign else { if (previousIncorrectSign != predictedSign) { timer = DateTime.Now; previousIncorrectSign = predictedSign; } else if (DateTime.Now - timer > TimeSpan.FromSeconds(2.0f)) { CheckEquality(predictedSign); timer = DateTime.Now; predictedSign = null; previousIncorrectSign = null; } } break; } } } else if (feedbackProgress != null) { feedbackProgress.value = 0.0f; } } /// /// Function to check equality between the current sign and the sign that the model predicted, if they are equal then the next sign is fetched. /// /// private void CheckEquality(string predicted) { if (predicted == course.theme.learnables[currentWordIndex].name) { NextSign(); } } }