WeSign/unity-application
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 6 Wiki Activity

Resolve WES-144 "Feedback justsign"

Browse Source
This commit is contained in:
Helena Van Breugel
2023-04-08 20:28:42 +00:00
parent 73c4756f19
commit 2b71bde592
56 changed files with 2966 additions and 1262 deletions
Download Patch File Download Diff File Expand all files Collapse all files

382
Assets/JustSign/Scripts/JustSignController.cs
View File

@@ -1,3 +1,4 @@
using DigitalRuby.Tween;
using System;
using System.Collections;
using System.Collections.Generic;
@@ -5,22 +6,18 @@ using System.Linq;
using TMPro;
using UnityEngine;
using UnityEngine.UI;
using Random = UnityEngine.Random;
/// <summary>
/// Contains all game logic for the JustSign game
/// </summary>
public class JustSignController : MonoBehaviour
{
public class JustSignController : AbstractFeedback
{
/// <summary>
/// All of the words that can be used in this session
/// </summary>
private List<Learnable> words = new List<Learnable>();
/// <summary>
/// The canvas containing all components
/// </summary>
public Canvas canvas;
/// <summary>
/// The input field where the user can type his or her answer
/// </summary>
@@ -29,7 +26,7 @@ public class JustSignController : MonoBehaviour
/// <summary>
/// The feedback on the timing
/// </summary>
public TMP_Text feedBack;
public TMP_Text timingFeedback;
/// <summary>
/// The current score
@@ -57,11 +54,50 @@ public class JustSignController : MonoBehaviour
/// </summary>
private Song currentSong;
/// <summary>
/// Reference to the perfect hitzone
/// </summary>
public RectTransform hitZonePerfect;
/// <summary>
/// The zone that the player should be hitting with his or her inputs
/// Reference to the good hitzone
/// </summary>
public GameObject hitZone;
public RectTransform hitZoneGood;
/// <summary>
/// Reference to the meh hitzone
/// </summary>
public RectTransform hitZoneMeh;
/// <summary>
/// Reference to the webcam
/// </summary>
public RawImage webcamScreen;
/// <summary>
/// Score obtained when getting a perfect hit
/// </summary>
private int perfectScore = 50;
/// <summary>
/// Score obtained when getting a good hit
/// </summary>
private int goodScore = 20;
/// <summary>
/// Score obtained when getting a meh hit
/// </summary>
private int mehScore = 10;
/// <summary>
/// Score obtained when getting a terrible hit
/// </summary>
private int terribleScore = -3;
/// <summary>
/// Score obtained when symbol goes offscreen
/// </summary>
private int offscreenScore = -5;
/// <summary>
/// Symbol prefab
@@ -88,75 +124,25 @@ public class JustSignController : MonoBehaviour
/// </summary>
private List<GameObject> activeSymbols = new List<GameObject>();
/// <summary>
/// The current score
/// </summary>
private int score;
/// <summary>
/// Have the symbols started spawning or not
/// </summary>
private bool gameIsActive = false;
/// <summary>
/// Width and height of the symbols
/// </summary>
private int symbolSize = 280;
/// <summary>
/// Controls movement speed of symbols (higher -> faster)
/// </summary>
private int moveSpeed = 200;
private int moveSpeed = 100;
/// <summary>
/// Starting X-coordinate of a symbol = (-1920 - symbolsize) / 2
/// </summary>
private int trackX = -1100;
private int trackX = 1920 / 2;
/// <summary>
/// Starting Y-coordinate of a symbol
/// </summary>
private int trackY = -200;
/// <summary>
/// Max distance from hit zone to get perfect score
/// </summary>
private int perfectBoundary = 10;
/// <summary>
/// Score obtained when getting a perfect hit
/// </summary>
private int perfectScore = 5;
/// <summary>
/// Max distance from hit zone to get good score
/// </summary>
private int goodBoundary = 120;
/// <summary>
/// Score obtained when getting a good hit
/// </summary>
private int goodScore = 3;
/// <summary>
/// Max distance from hit zone to get meh score
/// </summary>
private int mehBoundary = 200;
/// <summary>
/// Score obtained when getting a meh hit
/// </summary>
private int mehScore = 1;
/// <summary>
/// Score obtained when getting a terrible hit
/// </summary>
private int terribleScore = -3;
/// <summary>
/// Score obtained when symbol goes offscreen
/// </summary>
private int offscreenScore = -5;
private int trackY = 0;
/// <summary>
/// Time at which the last symbol was spawned
@@ -258,17 +244,88 @@ public class JustSignController : MonoBehaviour
/// </summary>
public GameObject gameEndedPanel;
/// <summary>
/// Reference to the feedback field
/// </summary>
public TMP_Text feedbackText;
/// <summary>
/// Reference to the progress bar
/// </summary>
public Slider feedbackProgressBar;
/// <summary>
/// Reference to the progress bar image, so we can add fancy colors
/// </summary>
public Image feedbackProgressImage;
/// <summary>
/// Sprite shown when perfect score
/// </summary>
public Sprite perfectSprite;
/// <summary>
/// Sprite shown when good score
/// </summary>
public Sprite goodSprite;
/// <summary>
/// Sprite shown when meh score
/// </summary>
public Sprite mehSprite;
/// <summary>
/// Sprite shown when terrible score (too soon)
/// </summary>
public Sprite terribleSprite;
/// <summary>
/// Sprite shown when sign leaves screen
/// </summary>
public Sprite tooLateSprite;
/// <summary>
/// Reference to display the feedback image
/// </summary>
public Image imageFeedback;
/// <summary>
/// Message to display when there is no model
/// </summary>
public GameObject previewMessage;
/// <summary>
/// Reference to the score, feedback and image
/// </summary>
public GameObject userFeedback;
/// <summary>
/// Start is called before the first frame update
/// </summary>
public void Start()
{
currentTheme = minigame.themeList.themes[minigame.themeList.currentThemeIndex];
signPredictor.SetModel(currentTheme.modelIndex);
signPredictor.SwapScreen(webcamScreen);
AddSelfAsListener();
StartController();
}
/// <summary>
/// Holds the game-specific logic to start the controller
/// </summary>
public void StartController()
{
userFeedback.SetActive(currentTheme.modelIndex != ModelIndex.NONE);
previewMessage.SetActive(currentTheme.modelIndex == ModelIndex.NONE);
perfectSigns = 0;
goodSigns = 0;
mehSigns = 0;
terribleSigns = 0;
incorrectSigns = 0;
score = 0;
timingFeedback.text = "";
imageFeedback.sprite = minigame.thumbnail;
gameEndedPanel.SetActive(false);
// Create entry in current user for keeping track of progress
user = UserList.GetCurrentUser();
@@ -279,10 +336,9 @@ public class JustSignController : MonoBehaviour
progress.minigameIndex = minigame.index;
user.AddMinigameProgress(progress);
}
PersistentDataController.GetInstance().Save();
UserList.Save();
scoreDisplay.text = "Score: " + score.ToString();
currentTheme = minigame.themeList.themes[minigame.themeList.currentThemeIndex];
scoreDisplay.text = $"Score: {CalculateScore()}";
words.AddRange(currentTheme.learnables);
currentSong = songList.songs[songList.currentSongIndex];
AudioSource.PlayClipAtPoint(currentSong.song, Vector3.zero, 1.0f);
@@ -309,63 +365,21 @@ public class JustSignController : MonoBehaviour
{
if (gameIsActive)
{
int matchedSymbolIndex = -1;
for (int i = 0; i < activeWords.Count; i++)
{
if (activeWords[i].ToLower() == answerField.text.ToLower())
{
matchedSymbolIndex = i;
}
}
// Destroy the oldest symbol if the current input matches it
if (matchedSymbolIndex >= 0)
{
int difference = Math.Abs((int)(activeSymbols[matchedSymbolIndex].transform.position.x - hitZone.transform.position.x));
if (difference < perfectBoundary)
{
feedBack.text = "Perfect!";
perfectSigns++;
score += perfectScore;
}
else if (difference < goodBoundary)
{
feedBack.text = "Good!";
goodSigns++;
score += goodScore;
}
else if (difference < mehBoundary)
{
feedBack.text = "Meh...";
mehSigns++;
score += mehScore;
}
else
{
feedBack.text = "Terrible!";
terribleSigns++;
score += terribleScore;
}
DestroySymbolAt(matchedSymbolIndex);
answerField.text = "";
}
// Destroy the oldest symbol if it leaves the screen
if (activeSymbols.Count > 0)
{
if (activeSymbols[0].GetComponent<RectTransform>().localPosition.x > -trackX)
if (activeSymbols[0].GetComponent<RectTransform>().localPosition.x > trackX)
{
DestroySymbolAt(0);
incorrectSigns++;
feedBack.text = "Te laat!";
score += offscreenScore;
timingFeedback.text = $"Te laat! \n {offscreenScore}";
imageFeedback.sprite = tooLateSprite;
}
}
// Spawn new symbol every spawn period
float currentTime = Time.time;
if (currentTime - lastSpawn > currentSong.spawnPeriod && lastSymbolTime > currentTime)
if (currentTime - lastSpawn > 2*currentSong.spawnPeriod && lastSymbolTime > currentTime)
{
lastSpawn = currentTime;
SpawnNewSymbol();
@@ -384,10 +398,19 @@ public class JustSignController : MonoBehaviour
rectTransform.localPosition = new Vector3(rectTransform.localPosition.x + Time.deltaTime * moveSpeed, trackY, 0);
}
scoreDisplay.text = "Score: " + score.ToString();
scoreDisplay.text = $"Score: {CalculateScore()}";
}
}
/// <summary>
/// Calculate the score
/// </summary>
/// <returns>The calculated score</returns>
public int CalculateScore()
{
return goodSigns*goodScore + perfectSigns*perfectScore + mehScore*mehSigns + terribleScore*terribleSigns + incorrectSigns*offscreenScore;
}
/// <summary>
/// Display Scoreboard + Metrics
/// </summary>
@@ -433,23 +456,20 @@ public class JustSignController : MonoBehaviour
}
}
Learnable newLearnable = words[unusedWordIndices[UnityEngine.Random.Range(0, unusedWordIndices.Count)]];
Learnable newLearnable = words[unusedWordIndices[Random.Range(0, unusedWordIndices.Count)]];
string nextSymbol = newLearnable.name;
GameObject newSymbolObject = GameObject.Instantiate(symbolPrefab, symbolContainer);
GameObject newSymbolObject = GameObject.Instantiate(symbolPrefab, new Vector3(0, trackY, 0), Quaternion.identity, symbolContainer);
// Dynamically load appearance
Image image = newSymbolObject.GetComponent<Image>();
Image image = newSymbolObject.transform.Find("Image").GetComponent<Image>();
image.sprite = newLearnable.image;
image.rectTransform.sizeDelta = new Vector2(symbolSize, symbolSize);
// Place the word that the symbol represents under the image
TMP_Text text = newSymbolObject.GetComponentInChildren<TMP_Text>();
text.text = nextSymbol;
text.color = Color.black;
text.rectTransform.localPosition = new Vector3(0, -160, 0);
activeWords.Add(nextSymbol);
activeWords.Add(nextSymbol.ToUpper());
activeSymbols.Add(newSymbolObject);
}
@@ -459,7 +479,7 @@ public class JustSignController : MonoBehaviour
public void SaveScores()
{
// Calculate new score
int newScore = this.score;
int newScore = CalculateScore();
// Save the score as a tuple: < int score, string time ago>
Score score = new Score();
@@ -513,7 +533,7 @@ public class JustSignController : MonoBehaviour
lpmText.text = (60f * correctSigns / duration).ToString("#") + " GPM";
// Score
scoreText.text = "Score: " + score.ToString();
scoreText.text = $"Score: {CalculateScore()}";
}
/// <summary>
@@ -616,4 +636,120 @@ public class JustSignController : MonoBehaviour
}
}
/// <summary>
/// The updateFunction that is called when new probabilities become available
/// </summary>
/// <returns></returns>
protected override IEnumerator UpdateFeedback()
{
// Get the predicted sign
if (signPredictor != null && signPredictor.learnableProbabilities != null && gameIsActive)
{
// Get highest predicted sign
string predictedSign = signPredictor.learnableProbabilities.Aggregate((a, b) => a.Value > b.Value ? a : b).Key;
float accuracy = signPredictor.learnableProbabilities[predictedSign];
// vvv TEMPORARY STUFF vvv
if (predictedSign == "J" && accuracy <= 0.97f)
{
predictedSign = signPredictor.learnableProbabilities.Aggregate((x, y) => x.Value > y.Value && x.Key != "J" ? x : y).Key;
}
accuracy = signPredictor.learnableProbabilities[predictedSign];
// ^^^ TEMPORARY STUFF ^^^
Learnable predSign = currentTheme.learnables.Find(l => l.name == predictedSign);
if (feedbackText != null && feedbackProgressImage != null)
{
Color col;
if (accuracy > predSign.thresholdPercentage)
{
feedbackText.text = $"Herkent '{predictedSign}'";
col = new Color(0x8b / 255.0f, 0xd4 / 255.0f, 0x5e / 255.0f);
}
else if (accuracy > 0.9 * predSign.thresholdPercentage)
{
feedbackText.text = $"Lijkt op '{predictedSign}'";
col = new Color(0xf2 / 255.0f, 0x7f / 255.0f, 0x0c / 255.0f);
}
else
{
feedbackText.text = "Detecteren...";
col = new Color(0xf5 / 255.0f, 0x49 / 255.0f, 0x3d / 255.0f);
}
feedbackText.color = col;
feedbackProgressImage.color = col;
float oldValue = feedbackProgressBar.value;
// use an exponential scale
float newValue = Mathf.Exp(4 * (Mathf.Clamp(accuracy / predSign.thresholdPercentage, 0.0f, 1.0f) - 1.0f));
feedbackProgressBar.gameObject.Tween("FeedbackUpdate", oldValue, newValue, 0.2f, TweenScaleFunctions.CubicEaseInOut, (t) =>
{
if (feedbackProgressBar != null)
{
feedbackProgressBar.value = t.CurrentValue;
}
});
}
if (accuracy > predSign.thresholdPercentage)
{
int matchedSymbolIndex = activeWords.IndexOf(predictedSign.ToUpper());
// Destroy the oldest symbol if the current input matches it
if (0 <= matchedSymbolIndex)
{
float x = activeSymbols[matchedSymbolIndex].transform.localPosition.x;
// parameters to define the Perfect hit zone
float perfectRange = hitZonePerfect.sizeDelta.x;
float perfectCenter = hitZonePerfect.localPosition.x;
// parameters to define the Good hit zone
float goodRange = hitZoneGood.sizeDelta.x;
float goodCenter = hitZoneGood.localPosition.x;
// parameters to define the Meh hit zone
float mehRange = hitZoneMeh.sizeDelta.x;
float mehCenter = hitZoneMeh.localPosition.x;
if (perfectCenter - perfectRange / 2 <= x && x <= perfectCenter + perfectRange / 2)
{
timingFeedback.text = $"Perfect! \n +{perfectScore}";
imageFeedback.sprite = perfectSprite;
perfectSigns++;
timingFeedback.color = new Color(0x8b / 255.0f, 0xd4 / 255.0f, 0x5e / 255.0f);
}
else if (goodCenter - goodRange / 2 <= x && x <= goodCenter + goodRange / 2)
{
timingFeedback.text = $"Goed \n +{goodScore}";
imageFeedback.sprite = goodSprite;
goodSigns++;
timingFeedback.color = new Color(0xf7 / 255.0f, 0xad / 255.0f, 0x19 / 255.0f);
}
else if (mehCenter - mehRange / 2 <= x && x <= mehCenter + mehRange / 2)
{
timingFeedback.text = $"Bijna... \n +{mehScore}";
imageFeedback.sprite = mehSprite;
mehSigns++;
timingFeedback.color = new Color(0xf2 / 255.0f, 0x7f / 255.0f, 0x0c / 255.0f);
}
else
{
timingFeedback.text = $"Te vroeg! \n {terribleScore}";
imageFeedback.sprite = terribleSprite;
terribleSigns++;
timingFeedback.color = new Color(0xf5 / 255.0f, 0x49 / 255.0f, 0x3d / 255.0f);
}
DestroySymbolAt(matchedSymbolIndex);
}
}
}
else if (feedbackProgressBar != null)
{
feedbackProgressBar.value = 0.0f;
}
yield return null;
}
}