Sprint 4

Assets/Courses/Scripts/CourseScripts.asmdef

@@ -6,7 +6,9 @@
         "AccountsScripts",
         "InterfacesScripts",
         "SignPredictor",
-        "Unity.Barracuda"
+        "NatML.ML",
+        "Tween",
+        "ArchitectureScripts"
     ],
     "includePlatforms": [],
     "excludePlatforms": [],

Assets/Courses/Scripts/CoursesController.cs

@@ -0,0 +1,571 @@
+using DigitalRuby.Tween;
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Linq;
+using TMPro;
+using UnityEngine;
+using UnityEngine.UI;
+using UnityEngine.Video;
+
+/// <summary>
+/// TemplateCourse scene manager
+/// </summary>
+public class CoursesController : AbstractFeedback
+{
+    /// <summary>
+    /// Reference to the objet holding the title
+    /// </summary>
+    public TMP_Text courseTitle;
+
+    /// <summary>
+    /// The current user
+    /// </summary>
+    private User user;
+
+    /// <summary>
+    /// Current user progress for this course
+    /// </summary>
+    private PersistentDataController.SavedCourseProgress progress = null;
+
+    /// <summary>
+    /// ScriptableObject with list of all courses
+    /// </summary>
+    public CourseList courselist;
+
+    /// <summary>
+    /// Reference to Course ScriptableObject
+    /// </summary>
+    private Course course;
+
+    /// <summary>
+    /// Index of the current word/letter in the course.learnables list
+    /// </summary>
+    private int currentWordIndex = 0;
+
+    /// <summary>
+    /// This holds the amount of words in the course
+    /// </summary>
+    private int maxWords;
+
+    /// <summary>
+    /// The "finished" screen
+    /// </summary>
+    public GameObject ResultPanel;
+
+    /// <summary>
+    /// Reference to the title on the results panel
+    /// </summary>
+    public TMP_Text ResultsTitle;
+
+    /// <summary>
+    /// Reference to the description on the results panel
+    /// </summary>
+    public TMP_Text ResultsDecription;
+
+    /// <summary>
+    /// Button to go back to courses list
+    /// </summary>
+    public Button CoursesButton;
+
+    /// <summary>
+    /// DateTime containint the start moment
+    /// </summary>
+    private DateTime startMoment;
+
+    /// <summary>
+    /// Reference to the timeSpent UI
+    /// </summary>
+    public TMP_Text timeSpent;
+
+    /// <summary>
+    /// Reference to the feedback field
+    /// </summary>
+    private TMP_Text feedbackText;
+
+    /// <summary>
+    /// Reference to the progress bar
+    /// </summary>
+    private Slider feedbackProgress;
+
+    /// <summary>
+    /// Reference to the progress bar image, so we can add fancy colors
+    /// </summary>
+    private Image feedbackProgressImage;
+
+    public VideoPlayer videoPlayer;
+
+    /// <summary>
+    /// Timer to keep track of how long a incorrect sign is performed
+    /// </summary>
+    protected DateTime timer;
+
+    /// <summary>
+    /// Current predicted sign
+    /// </summary>
+    protected string predictedSign = null;
+
+    /// <summary>
+    /// Previous incorrect sign, so we can keep track whether the user is wrong or the user is still changing signs
+    /// </summary>
+    protected string previousIncorrectSign = null;
+
+    /// <summary>
+    /// Keeps track of what type of panel is currently being used
+    /// </summary>
+    protected int panelId = 0;
+
+    /// <summary>
+    /// Boolean used to check whether the user has already answered the question
+    /// </summary>
+    private bool hasAnswered = false;
+
+    /// <summary>
+    /// Boolean used to check whether  SlideIn animation is playing
+    /// </summary>
+    private bool isNextSignInTransit = false;
+
+    /// <summary>
+    /// Reference to course progress bar
+    /// </summary>
+    public SlicedSlider progressBar;
+
+    /// <summary>
+    /// Reference to the animator of the confetti animation
+    /// </summary>
+    public Animator confettiAnimation;
+
+    /// <summary>
+    /// Panel with video&image prefab
+    /// </summary>
+    public GameObject panelSignWithVideoAndImagePrefab;
+
+    /// <summary>
+    /// Panel with image prefab
+    /// </summary>
+    public GameObject panelSignWithImagePrefab;
+
+    /// <summary>
+    /// Panel with multiplechoice prefab
+    /// </summary>
+    public GameObject panelMultipleChoicePrefab;
+
+    /// <summary>
+    /// Reference to the canvas to put the panels into
+    /// </summary>
+    public Transform canvas;
+
+    /// <summary>
+    /// Reference to the previous panel, 
+    /// so it can be deleted when its done playing its exit animation
+    /// </summary>
+    private GameObject previousPanel = null;
+
+    /// <summary>
+    /// This function is called when the script is initialised.
+    /// It inactivatis the popup, finds a webcam to use and links it via the WebcamTexture to the display RawImage.
+    /// It takes the correct course from the courselist, using the courseIndex.
+    /// Then it checks whether or not the User has started the course yet, to possibly create a new progress atribute for the course.
+    /// Then it sets up the course-screen to display relevant information from the course-scriptable.
+    /// </summary>
+    void Start()
+    {
+        StartCourseController();
+        signPredictor.SetSignsList(GetSignsList());
+        signPredictor.SetModel(course.theme.modelIndex);
+        AddSelfAsListener();
+    }
+
+    /// <summary>
+    /// Fetches all the strings of the signs of the course
+    /// </summary>
+    /// <returns>The signsList that needs to be passed to the signPredictor</returns>
+    private List<string> GetSignsList()
+    {
+        List<string> signsList = new List<string>();
+        foreach (Learnable learnable in course.theme.learnables)
+        {
+            signsList.Add(learnable.name);
+        }
+
+        return signsList;
+    }
+
+    /// <summary>
+    /// Holds the course-specific logic to start the controller, it is seperated to allow the course to be reset (if that would become needed)
+    /// </summary>
+    public void StartCourseController()
+    {
+        // Setting up course
+        course = courselist.courses[courselist.currentCourseIndex];
+        maxWords = course.theme.learnables.Count;
+
+        // Reload from disk (course may be reset)
+        PersistentDataController.GetInstance().Load();
+        // Create entry in current user for keeping track of progress
+        user = UserList.GetCurrentUser();
+        progress = user.GetCourseProgress(course.index);
+        if (progress == null)
+        {
+            progress = new PersistentDataController.SavedCourseProgress();
+            progress.courseIndex = course.index;
+            int index = 0;
+            foreach (Learnable learnable in course.theme.learnables)
+            {
+                progress.AddLearnable(learnable.name, index++);
+            }
+            user.AddCourseProgress(progress);
+        }
+        UserList.Save();
+        progressBar.fillAmount = progress.progress;
+
+        currentWordIndex = 0;
+        previousPanel = SetupPanel();
+
+        // Hide the result panel
+        ResultPanel.SetActive(false);
+        // Set the startTime
+        startMoment = DateTime.Now;
+    }
+
+    /// <summary>
+    /// Fetch the next sign and its panel type
+    /// </summary>
+    /// <returns>A tuple of {next sign index, panel type}</returns>
+    /// <remarks>
+    /// The different panel types:<br></br>
+    /// 0 : panelSignWithVideoAndImagePrefab<br></br>
+    /// 1 : panelMultipleChoicePrefab<br></br>
+    /// 2 : panelSignWithImagePrefab
+    /// </remarks>
+    private Tuple<int, int> FetchSign()
+    {
+        PersistentDataController.SavedLearnableProgress learnable = progress.GetRandomLearnable();
+        int panelChosen;
+        if (course.theme.modelIndex == ModelIndex.NONE)
+        {
+            // only multiple choice works in preview mode
+            panelChosen = 1;
+        }
+        else if (learnable.progress > 2.0f)
+        {
+            panelChosen = 2;
+        }
+        else if (learnable.progress > 1.0f)
+        {
+            panelChosen = 1;
+        }
+        else
+        {
+            panelChosen = 0;
+        }
+        return Tuple.Create(learnable.index, panelChosen);
+    }
+
+    /// <summary>
+    /// This function is called when the next-sign button is pressed.
+    /// It increased the wordindex and fetches new videos/images if index<max, because then the coure is not fincished yet.
+    /// If the maximum is reached, finishcourse is called to save the "finished" progress to the user.
+    /// </summary>
+    public void NextSign()
+    {
+        // This function is also called (async) when pressing the 'Gebaar overslaan' button,
+        // so check for condition so we don't skip multiple signs
+        if (isNextSignInTransit || maxWords <= progress.completedLearnables)
+            return;
+
+        progress.progress = (float)progress.completedLearnables / (float)maxWords;
+        progressBar.fillAmount = progress.progress;
+
+        // Update UI if course is not finished yet
+        if (progress.completedLearnables < maxWords)
+        {
+            // Set next sign/video/image
+            StartCoroutine(CRNextSign());
+        }
+        // Finish course and record progress
+        if (progress.completedLearnables == maxWords)
+        {
+            FinishCourse();
+        }
+    }
+
+    /// <summary>
+    /// Coroutine for going to the next sign
+    /// </summary>
+    /// <returns></returns>
+    private IEnumerator CRNextSign()
+    {
+        isNextSignInTransit = true;
+        GameObject newPanel = SetupPanel();
+        previousPanel.transform.SetAsFirstSibling();
+        newPanel.GetComponent<Animator>().SetTrigger("Slide Panel In");
+        if (previousPanel != null)
+            previousPanel.GetComponent<Animator>().SetTrigger("Slide Panel Out");
+
+        yield return new WaitForSeconds(1.0f);
+
+        confettiAnimation.ResetTrigger("Display Confetti");
+        GameObject.Destroy(previousPanel);
+        previousPanel = newPanel;
+        hasAnswered = false;
+        isNextSignInTransit = false;
+    }
+
+    /// <summary>
+    /// Setup a new panel
+    /// </summary>
+    /// <returns>Reference to the GameObject of the panel</returns>
+    private GameObject SetupPanel()
+    {
+        (currentWordIndex, panelId) = FetchSign().ToValueTuple();
+        switch (panelId)
+        {
+            case 0:
+                {
+                    GameObject panel = GameObject.Instantiate(panelSignWithVideoAndImagePrefab, canvas);
+                    panel.transform.SetAsFirstSibling();
+
+                    PanelWithVideoAndImage script = panel.GetComponent<PanelWithVideoAndImage>();
+                    script.signs = course.theme.learnables;
+                    script.currentSignIndex = currentWordIndex;
+                    script.isPreview = (course.theme.modelIndex == ModelIndex.NONE);
+                    script.videoPlayer = videoPlayer;
+                    feedbackProgress = script.feedbackProgressBar;
+                    feedbackProgressImage = script.feedbackProgressImage;
+                    feedbackText = script.feedbackText;
+                    script.Display();
+                    signPredictor.SwapScreen(script.webcamScreen);
+                    courseTitle.text = "Voer het gebaar uit voor \"" + course.theme.learnables[currentWordIndex].name + "\"";
+                    return panel;
+                }
+            case 1:
+                {
+                    GameObject panel = GameObject.Instantiate(panelMultipleChoicePrefab, canvas);
+                    panel.transform.SetAsFirstSibling();
+
+                    PanelMultipleChoice script = panel.GetComponent<PanelMultipleChoice>();
+                    script.signs = course.theme.learnables;
+                    script.currentSignIndex = currentWordIndex;
+                    script.videoPlayer = videoPlayer;
+                    script.courseController = this;
+                    script.progress = progress;
+                    script.isFingerSpelling = course.theme.title == "Handalfabet";
+                    script.Display();
+                    signPredictor.SwapScreen(script.webcamScreen);
+                    courseTitle.text = "Welk gebaar zie je hier?";
+                    return panel;
+                }
+            case 2:
+                {
+                    GameObject panel = GameObject.Instantiate(panelSignWithImagePrefab, canvas);
+                    panel.transform.SetAsFirstSibling();
+
+                    PanelWithImage script = panel.GetComponent<PanelWithImage>();
+                    script.signs = course.theme.learnables;
+                    script.currentSignIndex = currentWordIndex;
+                    script.isPreview = (course.theme.modelIndex == ModelIndex.NONE);
+                    feedbackProgress = script.feedbackProgressBar;
+                    feedbackProgressImage = script.feedbackProgressImage;
+                    feedbackText = script.feedbackText;
+                    script.Display();
+                    signPredictor.SwapScreen(script.webcamScreen);
+                    courseTitle.text = "Voer het gebaar uit voor \"" + course.theme.learnables[currentWordIndex].name + "\"";
+                    return panel;
+                }
+        }
+        return null;
+    }
+
+    /// <summary>
+    /// finishcourse is called to save the "finished" progress to the user.
+    /// </summary>
+    public void FinishCourse()
+    {
+        // Show the "finished" screen
+        ResultPanel.SetActive(true);
+
+        // Set the correct title
+        ResultsTitle.text = course.title + " voltooid!";
+
+        // Set the correct description
+        ResultsDecription.text = "Goed gedaan! Je kan nu spelletjes spelen met " + course.title + " om verder te oefenen!";
+
+        // Set the total time spent UI
+        TimeSpan time = DateTime.Now - startMoment;
+        timeSpent.text = time.ToString(@"hh\:mm\:ss");
+
+        // Link button
+        CoursesButton.onClick.AddListener(() => { SystemController.GetInstance().BackToPreviousScene(); });
+
+        progress.progress = 1.0f;
+        UserList.Save();
+    }
+
+    /// <summary>
+    /// The updateFunction that is called when new probabilities become available
+    /// </summary>
+    /// <returns></returns>
+    protected override IEnumerator UpdateFeedback()
+    {
+        // Check if the current word index is still in bounds, and if the current panel type is not multiple choice
+        if (currentWordIndex < course.theme.learnables.Count && panelId != 1 && !hasAnswered)
+        {
+            // Get current sign
+            Learnable sign = course.theme.learnables[currentWordIndex];
+            string currentSign = sign.name.ToUpper().Replace(" ", "-");
+
+            // Get the predicted sign
+            if (signPredictor != null && signPredictor.learnableProbabilities != null &&
+                currentSign != null && signPredictor.learnableProbabilities.ContainsKey(currentSign))
+            {
+                //Debug.Log($"{signPredictor.learnableProbabilities.Aggregate("", (t, e) => $"{t}{e.Key}={e.Value}, ")}");
+                float accCurrentSign = signPredictor.learnableProbabilities[currentSign];
+
+                // Get highest predicted sign
+                string predictedSign = signPredictor.learnableProbabilities.Aggregate((a, b) => a.Value > b.Value ? a : b).Key;
+                float accPredictSign = signPredictor.learnableProbabilities[predictedSign];
+                Learnable predSign = course.theme.learnables.Find(l => l.name.ToUpper().Replace(" ", "-") == predictedSign);
+
+                // If there is a feedback-object, we wil change its appearance
+                if (feedbackText != null && feedbackProgressImage != null)
+                {
+                    Color col;
+                    if (accCurrentSign > sign.thresholdPercentage)
+                    {
+                        feedbackText.text = "Goed";
+                        col = new Color(0x8b / 255.0f, 0xd4 / 255.0f, 0x5e / 255.0f);
+                    }
+                    else if (accCurrentSign > 0.9 * sign.thresholdPercentage)
+                    {
+                        feedbackText.text = "Bijna";
+                        col = new Color(0xf2 / 255.0f, 0x7f / 255.0f, 0x0c / 255.0f);
+                    }
+                    else if (accPredictSign > predSign.thresholdPercentage)
+                    {
+                        feedbackText.text = $"Verkeerde gebaar: '{predSign.name}'";
+                        col = new Color(0xf5 / 255.0f, 0x49 / 255.0f, 0x3d / 255.0f);
+                        accCurrentSign = 0.0f;
+                    }
+                    else
+                    {
+                        feedbackText.text = $"Detecteren ...";
+                        col = new Color(0xf5 / 255.0f, 0x49 / 255.0f, 0x3d / 255.0f);
+                        //accCurrentSign = 0.0f;
+                    }
+
+                    feedbackText.color = col;
+                    feedbackProgressImage.color = col;
+
+                    // Tween the feedback-bar
+                    float oldValue = feedbackProgress.value;
+                    // use an exponential scale 
+                    float newValue = Mathf.Exp(4 * (Mathf.Clamp(accCurrentSign / sign.thresholdPercentage, 0.0f, 1.0f) - 1.0f));
+                    feedbackProgress.gameObject.Tween("FeedbackUpdate", oldValue, newValue, 0.2f, TweenScaleFunctions.CubicEaseInOut, (t) =>
+                    {
+                        if (feedbackProgress != null)
+                        {
+                            feedbackProgress.value = t.CurrentValue;
+                        }
+                    });
+                }
+
+                // The internal logic for the courses
+                if (accPredictSign > sign.thresholdPercentage)
+                {
+                    // Correct sign
+                    if (predictedSign == currentSign)
+                    {
+                        yield return new WaitForSeconds(1.0f);
+                        NextSignIfCorrect(currentSign, predictedSign);
+                        timer = DateTime.Now;
+                        previousIncorrectSign = null;
+                        predictedSign = null;
+                    }
+                    // Incorrect sign
+                    else
+                    {
+                        if (previousIncorrectSign != predictedSign)
+                        {
+                            timer = DateTime.Now;
+                            previousIncorrectSign = predictedSign;
+                        }
+                        else if (predictedSign != null && currentSign != null &&
+                            (DateTime.Now - timer).TotalSeconds > 2.0f)
+                        {
+                            NextSignIfCorrect(currentSign, predictedSign);
+                            timer = DateTime.Now;
+                            predictedSign = null;
+                            previousIncorrectSign = null;
+                        }
+                    }
+                }
+            }
+            else if (feedbackProgress != null)
+            {
+                feedbackProgress.value = 0.0f;
+            }
+        }
+    }
+
+    /// <summary>
+    /// Function to check equality between the current sign and the sign that the model predicted, if they are equal then the next sign is fetched.
+    /// </summary>
+    /// <param name="predicted"></param>
+    public void NextSignIfCorrect(string current, string predicted)
+    {
+        if (!hasAnswered)
+        {
+            if (current == predicted)
+            {
+                hasAnswered = true;
+                var p = progress.learnables.Find((l) => l.name.ToUpper().Replace(" ", "-") == predicted);
+                progress.UpdateLearnable(p.name, 1.5f);
+                confettiAnimation.SetTrigger("Display Confetti");
+                StartCoroutine(WaitNextSign());
+            }
+            else
+            {
+                // currently ignore wrong signs as "J" doesn't work well enough
+            }
+        }
+
+    }
+
+    private IEnumerator WaitNextSign()
+    {
+        // Wait for 0.75 seconds
+        yield return new WaitForSeconds(0.75f);
+        NextSign();
+    }
+
+    /// <summary>
+    /// Function to check equality between the current sign and the sign that the model predicted, if they are equal then the next sign is fetched.
+    /// </summary>
+    /// <param name="predicted"></param>
+    public void NextSignMultipleChoice(string current, string predicted)
+    {
+        if (!hasAnswered)
+        {
+            hasAnswered = true;
+            if (current == predicted)
+            {
+                progress.UpdateLearnable(predicted, 1.5f);
+                confettiAnimation.SetTrigger("Display Confetti");
+            }
+            else
+            {
+                progress.UpdateLearnable(predicted, -1.0f);
+            }
+        }
+    }
+
+    /// <summary>
+    /// Callback for the 'back' button
+    /// </summary>
+    public void ReturnToActivityScreen()
+    {
+        UserList.Save();
+        SystemController.GetInstance().BackToPreviousScene();
+    }
+}

Assets/Courses/Scripts/PanelMultipleChoice.cs

@@ -0,0 +1,216 @@
+using System.Collections;
+using System.Collections.Generic;
+using TMPro;
+using UnityEngine;
+using UnityEngine.UI;
+using UnityEngine.Video;
+
+
+public class PanelMultipleChoice : MonoBehaviour
+{
+    /// <summary>
+    /// Reference to the webcam screen
+    /// </summary>
+    public RawImage webcamScreen;
+
+    /// <summary>
+    /// Video 'play' sprite
+    /// </summary>
+    public Sprite playSprite;
+
+    /// <summary>
+    /// Video 'pause' sprite
+    /// </summary>
+    public Sprite pauseSprite;
+
+    /// <summary>
+    /// Reference to its course controller to be able to go to call NextSignMultipleChoice to go to the next panel
+    /// </summary>
+    public CoursesController courseController;
+
+    /// <summary>
+    /// Reference to instructional video player
+    /// </summary>
+    public VideoPlayer videoPlayer;
+
+    /// <summary>
+    /// Refrence to the video play/pause button
+    /// </summary>
+    public Image playButton;
+
+    /// <summary>
+    /// Reference to the image for displaying the current words sprite
+    /// </summary>
+    public Transform optionContainer;
+
+    /// <summary>
+    /// Reference to the image for displaying the current words sprite, for a fingerspelling courses
+    /// </summary>
+    public Transform optionFingerspellingContainer;
+
+    /// <summary>
+    /// Reference to the option prefab
+    /// </summary>
+    public GameObject optionPrefab;
+
+    /// <summary>
+    /// Reference to the option prefab, for a fingerspelling course
+    /// </summary>
+    public GameObject optionFingerspellingPrefab;
+
+    /// <summary>
+    /// Reference to the saved progress
+    /// </summary>
+    public PersistentDataController.SavedCourseProgress progress;
+
+    /// <summary>
+    /// The current sign that will be displayed
+    /// </summary>
+    private Learnable currentSign;
+
+    /// <summary>
+    /// Reference to all signs in this course
+    /// </summary>
+    public List<Learnable> signs;
+
+    /// <summary>
+    /// Index of the current sign
+    /// </summary>
+    public int currentSignIndex;
+
+    /// <summary>
+    /// Boolean used to check whether the current course is a fingerspelling course
+    /// </summary>
+    public bool isFingerSpelling;
+
+    /// <summary>
+    /// Boolean used to check whether the user has already answered the question
+    /// </summary>
+    private bool hasAnswered = false;
+
+    /// <summary>
+    /// Get a list of wrongs answers
+    /// </summary>
+    /// <param name="notThisIndex">The index of the correct sign</param>
+    /// <returns></returns>
+    public List<Learnable> GetWrongOptions(int notThisIndex)
+    {
+        List<Learnable> randomSigns = new List<Learnable>();
+        // TODO: find more koosjer  way to do this
+        while (randomSigns.Count < 3)
+        {
+            int index = progress.GetRandomLearnable().index;
+            if (index != notThisIndex && !randomSigns.Contains(signs[index]))
+            {
+                randomSigns.Add(signs[index]);
+            }
+        }
+
+        return randomSigns;
+    }
+
+    /// <summary>
+    /// Update the display of this panel
+    /// </summary>
+    public void Display()
+    {
+        currentSign = signs[currentSignIndex];
+        videoPlayer.aspectRatio = VideoAspectRatio.FitInside;
+        videoPlayer.clip = currentSign.clip;
+        videoPlayer.Play();
+        // Gets three random selected signs from the list signs which are not equal to currentSign
+        List<Learnable> allOptions = GetWrongOptions(currentSignIndex);
+
+        // Add the correct sign at a random position in the list of all options
+        int randomIndex = UnityEngine.Random.Range(0, allOptions.Count + 1);
+        allOptions.Insert(randomIndex, currentSign);
+
+        var prefab = isFingerSpelling ? optionFingerspellingPrefab : optionPrefab;
+        var container = isFingerSpelling ? optionFingerspellingContainer : optionContainer;
+        foreach (Learnable option in allOptions)
+        {
+            GameObject multipleChoiceOption = GameObject.Instantiate(prefab, container);
+            if (!isFingerSpelling)
+                multipleChoiceOption.transform.Find("TextOption").GetComponent<TMP_Text>().text = option.name;
+            multipleChoiceOption.transform.Find("ImageOption").GetComponent<Image>().sprite = option.image;
+
+            Button button = multipleChoiceOption.GetComponent<Button>();
+            button.onClick.AddListener(() =>
+            {
+                if (!hasAnswered)
+                {
+                    courseController.NextSignMultipleChoice(currentSign.name, option.name);
+                    ShowAnswers(option.name);
+                    hasAnswered = true;
+                }
+            });
+        }
+
+    }
+
+    /// <summary>
+    /// Show the asnwers
+    /// </summary>
+    /// <param name="answerName">The name of the clicked sign</param>
+    public void ShowAnswers(string answerName)
+    {
+        if (answerName == currentSign.name)
+        {
+            courseController.courseTitle.text = "Correct!";
+        }
+        else
+        {
+            courseController.courseTitle.text = "Spijtig, fout";
+        }
+
+        var container = isFingerSpelling ? optionFingerspellingContainer : optionContainer;
+        for (int i = 0; i < container.childCount; i++)
+        {
+            // Get the i-th child object
+            Transform childTransform = container.GetChild(i);
+
+            // Change the background color of the Button
+            Color col = new Color(0xf5 / 255.0f, 0x49 / 255.0f, 0x3d / 255.0f);
+            if (childTransform.Find("ImageOption").GetComponent<Image>().sprite == currentSign.image)
+            {
+                col = new Color(0x8b / 255.0f, 0xd4 / 255.0f, 0x5e / 255.0f);
+            }
+            childTransform.GetComponent<Image>().color = col;
+        }
+        StartCoroutine(GoToNextScreen());
+    }
+
+    /// <summary>
+    /// Wait and go to the next sign
+    /// </summary>
+    /// <returns></returns>
+    private IEnumerator GoToNextScreen()
+    {
+        // Wait for 5 seconds
+        yield return new WaitForSeconds(1.5f);
+        courseController.NextSign();
+    }
+
+    /// <summary>
+    /// This function is called when the pause-button is pressed on the video.
+    /// It switches between playing and pausing the video.
+    /// It then makes the button invisible when the video is playing, or visible when it's paused.
+    /// </summary>
+    public void TogglePlayPause()
+    {
+        if (!videoPlayer.isPlaying)
+        {
+            // Play video and switch sprite of button
+            playButton.sprite = pauseSprite;
+            videoPlayer.Play();
+        }
+
+        else
+        {
+            // Pause video and and switch sprite of button
+            playButton.sprite = playSprite;
+            videoPlayer.Pause();
+        }
+
+    }
+}

Assets/Courses/Scripts/PanelMultipleChoice.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: e46fcdec22898f04eac0a7981baae26a
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Assets/Courses/Scripts/PanelWithImage.cs

@@ -0,0 +1,49 @@
+using System.Collections.Generic;
+using TMPro;
+using UnityEngine;
+using UnityEngine.UI;
+
+public class PanelWithImage : MonoBehaviour
+{
+    public GameObject feedbackProgressObject;
+    public GameObject previewMessage;
+    public bool isPreview;
+
+    /// <summary>
+    /// Reference to the image for displaying the current words sprite
+    /// </summary>
+    public Transform signImageContainer;
+
+    public GameObject signImagePrefab;
+
+    public RawImage webcamScreen;
+
+    /// <summary>
+    /// Reference to the feedback field
+    /// </summary>
+    public TMP_Text feedbackText;
+
+    /// <summary>
+    /// Reference to the progress bar
+    /// </summary>
+    public Slider feedbackProgressBar;
+
+    /// <summary>
+    /// Reference to the progress bar image, so we can add fancy colors
+    /// </summary>
+    public Image feedbackProgressImage;
+
+    public List<Learnable> signs;
+    public int currentSignIndex;
+
+    public void Display()
+    {
+        Learnable currentSign = signs[currentSignIndex];
+
+        feedbackProgressObject.SetActive(!isPreview);
+        previewMessage.SetActive(isPreview);
+
+        GameObject sprite = GameObject.Instantiate(signImagePrefab, signImageContainer);
+        sprite.GetComponent<Image>().sprite = currentSign.image;
+    }
+}

Assets/Courses/Scripts/PanelWithImage.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: 09eba136d66a8cd4abf6f6ebfd0006f4
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Assets/Courses/Scripts/PanelWithVideoAndImage.cs

@@ -0,0 +1,91 @@
+using System.Collections.Generic;
+using TMPro;
+using UnityEngine;
+using UnityEngine.UI;
+using UnityEngine.Video;
+
+public class PanelWithVideoAndImage : MonoBehaviour
+{
+    public GameObject feedbackProgressObject;
+    public GameObject previewMessage;
+    public bool isPreview;
+
+    public Sprite playSprite;
+    public Sprite pauseSprite;
+
+    /// <summary>
+    /// Reference to instructional video player
+    /// </summary>
+    public VideoPlayer videoPlayer;
+    public Image playButton;
+    public RawImage webcamScreen;
+
+    /// <summary>
+    /// Reference to the image for displaying the current words sprite
+    /// </summary>
+    public Transform signImageContainer;
+
+    public GameObject signImagePrefab;
+
+    /// <summary>
+    /// Reference to the feedback field
+    /// </summary>
+    public TMP_Text feedbackText;
+
+    /// <summary>
+    /// Reference to the progress bar
+    /// </summary>
+    public Slider feedbackProgressBar;
+
+    /// <summary>
+    /// Reference to the progress bar image, so we can add fancy colors
+    /// </summary>
+    public Image feedbackProgressImage;
+
+    public List<Learnable> signs;
+    public int currentSignIndex;
+
+    public void Display()
+    {
+        Learnable currentSign = signs[currentSignIndex];
+        videoPlayer.aspectRatio = VideoAspectRatio.FitInside;
+        videoPlayer.clip = currentSign.clip;
+        videoPlayer.Play();
+
+        feedbackProgressObject.SetActive(!isPreview);
+        previewMessage.SetActive(isPreview);
+
+        List<Sprite> sprites = new List<Sprite>() { currentSign.image };
+        if (currentSign.handGuide != null)
+            sprites.Add(currentSign.handGuide);
+
+        foreach (Sprite s in sprites)
+        {
+            GameObject sprite = GameObject.Instantiate(signImagePrefab, signImageContainer);
+            sprite.GetComponent<Image>().sprite = s;
+        }
+    }
+
+    /// <summary>
+    /// This function is called when the pause-button is pressed on the video.
+    /// It switches between playing and pausing the video.
+    /// It then makes the button invisible when the video is playing, or visible when it's paused.
+    /// </summary>
+    public void TogglePlayPause()
+    {
+        if (!videoPlayer.isPlaying)
+        {
+            // Play video and switch sprite of button
+            playButton.sprite = pauseSprite;
+            videoPlayer.Play();
+        }
+           
+        else
+        {
+            // Pause video and and switch sprite of button
+            playButton.sprite = playSprite;
+            videoPlayer.Pause();
+        }
+            
+    }
+}

Assets/Courses/Scripts/PanelWithVideoAndImage.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: 4631ff9751fcb1e4b8ef5451d2650537
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Assets/Courses/Scripts/TemplateCourse.cs

@@ -1,289 +0,0 @@
-using System;
-using TMPro;
-using Unity.Barracuda;
-using UnityEngine;
-using UnityEngine.UI;
-using UnityEngine.Video;
-
-/// <summary>
-/// TemplateCourse scene manager
-/// </summary>
-public class TemplateCourse : MonoBehaviour
-{
-    // vvv TEMPORARY STUFF vvv
-    public NNModel previewModel;
-    public GameObject feedbackProgressBar;
-    public GameObject previewMessage;
-    // ^^^ TEMPORARY STUFF ^^^
-
-    /// <summary>
-    /// Reference to instructional video player
-    /// </summary>
-    public VideoPlayer player;
-
-    /// <summary>
-    /// Reference to pause button
-    /// </summary>
-    public Button button;
-
-    /// <summary>
-    /// Reference to sprite for the pause button
-    /// </summary>
-    public Sprite pauseSprite;
-
-    /// <summary>
-    /// Reference to the image for displaying the current words sprite
-    /// </summary>
-    public Image wordImage;
-
-    /// <summary>
-    /// Reference to the text object for displaying the current word
-    /// </summary>
-    public TMP_Text title;
-
-    /// <summary>
-    /// Reference to user list to get current user
-    /// </summary>
-    public UserList userList;
-
-    /// <summary>
-    /// The current user
-    /// </summary>
-    private User user;
-
-    /// <summary>
-    /// Current user progress for this course
-    /// </summary>
-    private Progress progress = null;
-
-    /// <summary>
-    /// ScriptableObject with list of all courses
-    /// </summary>
-    public CourseList courselist;
-
-    /// <summary>
-    /// Reference to Course ScriptableObject
-    /// </summary>
-    private Course course;
-
-    /// <summary>
-    /// Index of the current word/letter in the course.learnables list
-    /// </summary>
-    private int currentWordIndex = 0;
-
-    /// <summary>
-    /// This holds the amount of words in the course
-    /// </summary>
-    private int maxWords;
-
-    /// <summary>
-    /// Number of correct words so far
-    /// (can be modified to a list or something like that to give better feedback)
-    /// </summary>
-    private int correctWords = 0;
-
-    /// <summary>
-    /// The "finished" screen
-    /// </summary>
-    public GameObject ResultPanel;
-
-    /// <summary>
-    /// Reference to the title on the results panel
-    /// </summary>
-    public TMP_Text ResultsTitle;
-
-    /// <summary>
-    /// Reference to the description on the results panel
-    /// </summary>
-    public TMP_Text ResultsDecription;
-
-    /// <summary>
-    /// Button to go back to courses list
-    /// </summary>
-    public Button CoursesButton;
-
-    /// <summary>
-    /// DateTime containint the start moment
-    /// </summary>
-    private DateTime startMoment;
-
-    /// <summary>
-    /// Reference to the timeSpent UI
-    /// </summary>
-    public TMP_Text timeSpent;
-
-    /// <summary>
-    /// Reference to the feedback script on the Feedback prefab
-    /// </summary>
-    public Feedback feedback;
-
-    /// <summary>
-    /// This function is called when the script is initialised.
-    /// It inactivatis the popup, finds a webcam to use and links it via the WebcamTexture to the display RawImage.
-    /// It takes the correct course from the courselist, using the courseIndex.
-    /// Then it checks whether or not the User has started the course yet, to possibly create a new progress atribute for the course.
-    /// Then it sets up the course-screen to display relevant information from the course-scriptable.
-    /// </summary>
-    void Awake()
-    {
-        // Setting up course
-        course = courselist.courses[courselist.currentCourseIndex];
-        feedback.signPredictor.model = course.theme.model;
-        maxWords = course.theme.learnables.Count;
-
-        // vvv TEMPORARY STUFF vvv
-        feedbackProgressBar.SetActive(course.theme.model != null);
-        previewMessage.SetActive(course.theme.model == null);
-        feedback.signPredictor.model = previewModel;
-        // ^^^ TEMPORARY STUFF ^^^
-
-        // Create entry in current user for keeping track of progress
-        userList.Load();
-        user = userList.GetCurrentUser();
-        progress = user.GetCourseProgress(course.index);
-        if (progress == null)
-        {
-            progress = new Progress();
-            progress.AddOrUpdate<CourseIndex>("courseIndex", course.index);
-            progress.AddOrUpdate<float>("courseProgress", -1.0f);
-            user.courses.Add(progress);
-        }
-        userList.Save();
-
-        // Force the videoplayer to add bars to preserve aspect ratio
-        player.aspectRatio = VideoAspectRatio.FitInside;
-
-        // Setup UI
-        button.image.sprite = pauseSprite;
-        title.text = course.title;
-        NextVideo();
-        NextImage();
-
-        // Hide the result panel
-        ResultPanel.SetActive(false);
-        // Set the startTime
-        startMoment = DateTime.Now;
-
-        // Set callbacks
-        feedback.getSignCallback = () =>
-        {
-            if (currentWordIndex < course.theme.learnables.Count)
-            {
-                return course.theme.learnables[currentWordIndex].name;
-            }
-            return null;
-        };
-        feedback.predictSignCallback = (sign) =>
-        {
-            if (sign == course.theme.learnables[currentWordIndex].name)
-            {
-                NextSign();
-            }
-        };
-    }
-
-    /// <summary>
-    /// This function uses the word_i integer to grab the correct video from the course.learnabels.
-    /// When it has this video, it will load it into the videoplayer and set it to start.
-    /// </summary>
-    private void NextVideo()
-    {
-        player.clip = course.theme.learnables[currentWordIndex].clip;
-        // This loads first frame, so that it can be used as a sort-of preview for the video
-        player.Play();
-        // As the video will start playiing -> hide button
-        Color col = button.image.color;
-        col.a = 0;
-        button.image.color = col;
-    }
-
-    /// <summary>
-    /// This function uses the word_i integer to grab the correct image from the course.learnabels.
-    /// Then it simply loads it into wordImage so that it can be displayed.
-    /// </summary>
-    private void NextImage()
-    {
-        wordImage.sprite = course.theme.learnables[currentWordIndex].image;
-    }
-
-    /// <summary>
-    /// This function is called when the pause-button is pressed on the video.
-    /// It switches between playing and pausing the video. 
-    /// It then makes the button invisible when the video is playing, or visible when it's paused.
-    /// </summary>
-    public void Pause()
-    {
-        if (!player.isPlaying)
-        {
-            // Play video and hide button
-            player.Play();
-            Color col = button.image.color;
-            col.a = 0;
-            button.image.color = col;
-        }
-        else
-        {
-            // Pause video and show button
-            player.Pause();
-            Color col = button.image.color;
-            col.a = 255;
-            button.image.color = col;
-        }
-    }
-
-    /// <summary>
-    /// This function is called when the next-sign button is pressed.
-    /// It increased the wordindex and fetches new videos/images if index<max, because then the coure is not fincished yet.
-    /// If the maximum is reached, finishcourse is called to save the "finished" progress to the user.
-    /// </summary>
-    public void NextSign()
-    {
-        // If the currentindex >= maxwords, it indicated that the course is already finished, running the next code is the meaningless.
-        if (currentWordIndex >= maxWords) { return; }
-
-        // Goto the next word/letter
-        currentWordIndex++;
-
-        // TODO: fix correct word count
-        correctWords++;
-        progress.AddOrUpdate<float>("courseProgress", (float)correctWords / (float)maxWords);
-        userList.Save();
-
-        // Update UI if course is not finished yet
-        if (currentWordIndex < maxWords)
-        {
-            NextVideo();
-            NextImage();
-        }
-        // Finish course and record progress
-        else
-        {
-            FinishCourse();
-        }
-    }
-
-    /// <summary>
-    /// finishcourse is called to save the "finished" progress to the user.
-    /// </summary>
-    public void FinishCourse()
-    {
-        // Show the "finished" screen
-        ResultPanel.SetActive(true);
-
-        // Set the correct title
-        ResultsTitle.text = course.title + " voltooid!";
-
-        // Set the correct description
-        ResultsDecription.text = "Goed gedaan! Je kan nu spelletjes spelen met " + course.title + " om verder te oefenen!";
-
-        // Set the total time spent UI
-        TimeSpan time = DateTime.Now - startMoment;
-        timeSpent.text = time.ToString(@"hh\:mm\:ss");
-
-        // Link button
-        CoursesButton.onClick.AddListener(() => { SystemController.GetInstance().BackToPreviousScene(); });
-
-        progress.AddOrUpdate<float>("courseProgress", 1f);
-        userList.Save();
-    }
-}