Score saber data V2 (#76)

* Now parsing v2 of score saber scrape.

SongBrowserPlugin/DataAccess/ScoreSaberDatabase.cs

@@ -1,4 +1,5 @@
-using SimpleJSON;
+using Newtonsoft.Json;
+using SimpleJSON;
 using SongBrowser.Logging;
 using System;
 using System.Collections.Generic;
@@ -7,101 +8,36 @@ using Logger = SongBrowser.Logging.Logger;
 
 namespace SongBrowser.DataAccess
 {
-    public class ScoreSaberDifficulty
+    public class ScoreSaberSong
     {
-        public string name;
-        public float pp;
-        public float star;
+        public string song { get; set; }
+        public string mapper { get; set; }
+        public List<ScoreSaberSongDifficultyStats> diffs { get; set; }
     }
 
-    public class ScoreSaberData
+    public class ScoreSaberSongDifficultyStats
     {
-        public string name;        
-        public Dictionary<String, ScoreSaberDifficulty> difficultyToSaberDifficulty = new Dictionary<string, ScoreSaberDifficulty>();
-        public float maxStar = 0;
-        public float maxPp = 0;
-        public string version;
-
-        public void AddDifficultyRating(string name, float pp, float star)
-        {
-            // assume list is newest->oldest, so always take first result.
-            if (difficultyToSaberDifficulty.ContainsKey(name))
-            {
-                return;
-            }
-            ScoreSaberDifficulty ppDifficulty = new ScoreSaberDifficulty
-            {
-                name = name,
-                star = star,
-                pp = pp
-            };
-            difficultyToSaberDifficulty.Add(ppDifficulty.name, ppDifficulty);
-
-            if (pp > maxPp)
-                maxPp = pp;
-
-            if (star > maxStar)
-                maxStar = star;
-        }
+        public string diff { get; set; }
+        public long scores { get; set; }
+        public double stars { get; set; }
+        public double pp { get; set; }
     }
 
     public class ScoreSaberDataFile
     {
-        public Dictionary<String, ScoreSaberData> SongVersionToScoreSaberData;
+        public Dictionary<String, ScoreSaberSong> SongHashToScoreSaberData = null;
 
         public ScoreSaberDataFile(byte[] data)
         {
             Stopwatch timer = new Stopwatch();
             timer.Start();
 
-            SongVersionToScoreSaberData = new Dictionary<string, ScoreSaberData>();
-
             System.Globalization.NumberStyles style = System.Globalization.NumberStyles.AllowDecimalPoint;
 
             string result = System.Text.Encoding.UTF8.GetString(data);
 
-            JSONNode rootNode = JSON.Parse(result);
-            foreach (KeyValuePair<string, JSONNode> kvp in rootNode)
-            {                
-                JSONNode difficultyNodes = kvp.Value;
-                foreach (KeyValuePair<string, JSONNode> innerKvp in difficultyNodes)
-                {
-                    JSONNode node = innerKvp.Value;
-                    String version = node["key"];
-                    String name = node["name"];
-                    String difficultyName = node["difficulty"];
-                    if (difficultyName == "Expert+")
-                    {
-                        difficultyName = "ExpertPlus";
-                    }
-                    
-                    float pp = 0;
-                    float.TryParse(node["pp"], style, System.Globalization.CultureInfo.InvariantCulture, out pp);
-
-                    float starDifficulty = 0;
-                    float.TryParse(node["star"], style, System.Globalization.CultureInfo.InvariantCulture, out starDifficulty);
-
-                    ScoreSaberData ppData = null;                    
-                    if (!SongVersionToScoreSaberData.ContainsKey(version))
-                    {
-                        ppData = new ScoreSaberData
-                        {
-                            version = version,
-                            name = name
-                        };
-
-                        SongVersionToScoreSaberData.Add(version, ppData);
-                    }
-                    else
-                    {
-                        ppData = SongVersionToScoreSaberData[version];
-                    }
-
-                    // add difficulty  
-                    ppData.AddDifficultyRating(difficultyName, pp, starDifficulty);
-                }
-            }
-            
+            SongHashToScoreSaberData = JsonConvert.DeserializeObject<Dictionary<string, ScoreSaberSong>>(result);
+                        
             timer.Stop();
             Logger.Debug("Processing ScoreSaber data took {0}ms", timer.ElapsedMilliseconds);
         }

SongBrowserPlugin/DataAccess/SongBrowserModel.cs

@@ -1,4 +1,5 @@
 using SongBrowser.DataAccess;
+using SongBrowser.Internals;
 using SongBrowser.UI;
 using SongCore.OverrideClasses;
 using SongCore.Utilities;
@@ -30,7 +31,6 @@ namespace SongBrowser
         private Dictionary<string, int> _weights;
         private Dictionary<BeatmapDifficulty, int> _difficultyWeights;
         private Dictionary<string, ScrappedSong> _levelHashToDownloaderData = null;
-        private Dictionary<string, ScoreSaberData> _levelIdToScoreSaberData = null;
         private Dictionary<string, int> _levelIdToPlayCount;
         private Dictionary<string, string> _levelIdToSongVersion;
 
@@ -50,17 +50,6 @@ namespace SongBrowser
         }
 
         /// <summary>
-        /// Map LevelID to score saber data.
-        /// </summary>
-        public Dictionary<string, ScoreSaberData> LevelIdToScoreSaberData
-        {
-            get
-            {
-                return _levelIdToScoreSaberData;
-            }
-        }
-
-        /// <summary>
         /// Get the last selected (stored in settings) level id.
         /// </summary>
         public String LastSelectedLevelId
@@ -118,7 +107,6 @@ namespace SongBrowser
         {
             _levelIdToCustomLevel = new Dictionary<string, CustomPreviewBeatmapLevel>();
             _cachedLastWriteTimes = new Dictionary<String, double>();
-            _levelIdToScoreSaberData = new Dictionary<string, ScoreSaberData>();
             _levelIdToPlayCount = new Dictionary<string, int>();
             _levelIdToSongVersion = new Dictionary<string, string>();
 
@@ -244,7 +232,6 @@ namespace SongBrowser
             Logger.Info("Determining song download time and determining mappings took {0}ms", lastWriteTimer.ElapsedMilliseconds);
 
             // Update song Infos, directory tree, and sort
-            this.UpdateScoreSaberDataMapping();
             this.UpdatePlayCounts();
 
             // Check if we need to upgrade settings file favorites
@@ -341,52 +328,7 @@ namespace SongBrowser
                 _levelIdToPlayCount[levelData.levelID] += levelData.playCount;
             }
         }
-
-        /// <summary>
-        /// Parse the current pp data file.
-        /// Public so controllers can decide when to update it.
-        /// </summary>
-        public void UpdateScoreSaberDataMapping()
-        {
-            Logger.Trace("UpdateScoreSaberDataMapping()");
-
-            ScoreSaberDataFile scoreSaberDataFile = ScoreSaberDatabaseDownloader.ScoreSaberDataFile;
-
-            // bail
-            if (scoreSaberDataFile == null)
-            {
-                Logger.Warning("Score saber data is not ready yet...");
-                return;
-            }
-
-            foreach (var level in SongCore.Loader.CustomLevels)
-            {
-                // Skip
-                if (_levelIdToScoreSaberData.ContainsKey(level.Value.levelID))
-                {
-                    continue;
-                }
-
-                ScoreSaberData scoreSaberData = null;
-
-                // try to version match first
-                if (_levelIdToSongVersion.ContainsKey(level.Value.levelID))
-                {
-                    String version = _levelIdToSongVersion[level.Value.levelID];
-                    if (scoreSaberDataFile.SongVersionToScoreSaberData.ContainsKey(version))
-                    {
-                        scoreSaberData = scoreSaberDataFile.SongVersionToScoreSaberData[version];
-                    }
-                }
-
-                if (scoreSaberData != null)
-                {
-                    //Logger.Debug("{0} = {1}pp", level.songName, pp);
-                    _levelIdToScoreSaberData.Add(level.Value.levelID, scoreSaberData);
-                }
-            }
-        }
-
+        
         /// <summary>
         /// Map the downloader data for quick lookup.
         /// </summary>
@@ -759,8 +701,24 @@ namespace SongBrowser
         {
             Logger.Info("Sorting song list by performance points...");
 
+            if (ScoreSaberDatabaseDownloader.ScoreSaberDataFile == null)
+            {
+                return levels;
+            }
+
             return levels
-                .OrderByDescending(x => _levelIdToScoreSaberData.ContainsKey(x.levelID) ? _levelIdToScoreSaberData[x.levelID].maxPp : 0)
+                .OrderByDescending(x =>
+                {
+                    var hash = CustomHelpers.GetSongHash(x.levelID);
+                    if (ScoreSaberDatabaseDownloader.ScoreSaberDataFile.SongHashToScoreSaberData.ContainsKey(hash))
+                    {
+                        return ScoreSaberDatabaseDownloader.ScoreSaberDataFile.SongHashToScoreSaberData[hash].diffs.Max(y => y.pp);
+                    }
+                    else
+                    {
+                        return 0;
+                    }
+                })
                 .ToList();
         }
 
@@ -851,7 +809,7 @@ namespace SongBrowser
 
             return levelIds
                 .OrderByDescending(x => {
-                    var hash = x.levelID.Split('_')[2];
+                    var hash = CustomHelpers.GetSongHash(x.levelID);
                     if (_levelHashToDownloaderData.ContainsKey(hash))
                     {
                         return _levelHashToDownloaderData[hash].Upvotes;
@@ -864,7 +822,6 @@ namespace SongBrowser
                 .ToList();
         }
 
-
         /// <summary>
         /// Sorting by BeatSaver rating stat.
         /// </summary>
@@ -882,7 +839,7 @@ namespace SongBrowser
 
             return levelIds
                 .OrderByDescending(x => {
-                    var hash = x.levelID.Split('_')[2];
+                    var hash = CustomHelpers.GetSongHash(x.levelID);
                     if (_levelHashToDownloaderData.ContainsKey(hash))
                     {
                         return _levelHashToDownloaderData[hash].PlayedCount;
@@ -912,7 +869,7 @@ namespace SongBrowser
 
             return levelIds
                 .OrderByDescending(x => {
-                    var hash = x.levelID.Split('_')[2];
+                    var hash = CustomHelpers.GetSongHash(x.levelID);
                     if (_levelHashToDownloaderData.ContainsKey(hash))
                     {
                         return _levelHashToDownloaderData[hash].Rating;

SongBrowserPlugin/Internals/CustomHelpers.cs

@@ -73,5 +73,10 @@ namespace SongBrowser.Internals
             }
             return false;
         }
+
+        public static string GetSongHash(string levelId)
+        {
+            return levelId.Split('_')[2];
+        }
     }
 }

SongBrowserPlugin/SongBrowserApplication.cs

@@ -114,7 +114,6 @@ namespace SongBrowser
             Logger.Trace("OnScoreSaberDataDownloaded");
             try
             {
-                _songBrowserUI.Model.UpdateScoreSaberDataMapping();
                 if (_songBrowserUI.Model.Settings.sortMode == SongSortMode.PP)
                 {
                     _songBrowserUI.ProcessSongList();

SongBrowserPlugin/UI/Browser/SongBrowserUI.cs

@@ -518,6 +518,11 @@ namespace SongBrowser.UI
         /// </summary>
         public void ProcessSongList()
         {
+            if (!_uiCreated)
+            {
+                return;
+            }
+
             this._model.ProcessSongList(_beatUi.GetCurrentSelectedLevelPack());
         }
 
@@ -1027,21 +1032,32 @@ namespace SongBrowser.UI
         {
             Logger.Trace("RefreshScoreSaberData({0})", level.levelID);
 
+            if (ScoreSaberDatabaseDownloader.ScoreSaberDataFile == null)
+            {
+                return;
+            }
+
             BeatmapDifficulty difficulty = _beatUi.LevelDifficultyViewController.selectedDifficulty;
             string difficultyString = difficulty.ToString();
+            if (difficultyString.Equals("ExpertPlus"))
+            {
+                difficultyString = "Expert+";
+            }
             Logger.Debug(difficultyString);
 
             // Check for PP
             Logger.Debug("Checking if have info for song {0}", level.songName);
-            if (this._model.LevelIdToScoreSaberData.ContainsKey(level.levelID))
+            var hash = CustomHelpers.GetSongHash(level.levelID);
+            if (ScoreSaberDatabaseDownloader.ScoreSaberDataFile.SongHashToScoreSaberData.ContainsKey(hash))
             {
                 Logger.Debug("Checking if have difficulty for song {0} difficulty {1}", level.songName, difficultyString);
-                ScoreSaberData ppData = this._model.LevelIdToScoreSaberData[level.levelID];
-                if (ppData.difficultyToSaberDifficulty.ContainsKey(difficultyString))
+                ScoreSaberSong scoreSaberSong = ScoreSaberDatabaseDownloader.ScoreSaberDataFile.SongHashToScoreSaberData[hash];
+                ScoreSaberSongDifficultyStats scoreSaberSongDifficulty = scoreSaberSong.diffs.FirstOrDefault(x => String.Equals(x.diff, difficultyString));
+                if (scoreSaberSongDifficulty != null)
                 {
                     Logger.Debug("Display pp for song.");
-                    float pp = ppData.difficultyToSaberDifficulty[difficultyString].pp;
-                    float star = ppData.difficultyToSaberDifficulty[difficultyString].star;
+                    double pp = scoreSaberSongDifficulty.pp;
+                    double star = scoreSaberSongDifficulty.stars;
 
                     UIBuilder.SetStatButtonText(_ppStatButton, String.Format("{0:0.#}", pp));
                     UIBuilder.SetStatButtonText(_starStatButton, String.Format("{0:0.#}", star));
@@ -1054,8 +1070,8 @@ namespace SongBrowser.UI
             }
             else
             {
-                UIBuilder.SetStatButtonText(_ppStatButton, "?");
-                UIBuilder.SetStatButtonText(_starStatButton, "?");
+                UIBuilder.SetStatButtonText(_ppStatButton, "NA");
+                UIBuilder.SetStatButtonText(_starStatButton, "NA");
             }                
 
             Logger.Debug("Done refreshing score saber stats.");

SongBrowserPlugin/UI/ScoreSaberDatabaseDownloader.cs

@@ -11,7 +11,7 @@ namespace SongBrowser.UI
 {
     public class ScoreSaberDatabaseDownloader : MonoBehaviour
     {
-        public const String SCRAPED_SCORE_SABER_JSON_URL = "https://wes.ams3.cdn.digitaloceanspaces.com/beatstar/bssb.json";
+        public const String SCRAPED_SCORE_SABER_JSON_URL = "https://raw.githubusercontent.com/halsafar/beat-saber-scraped-data/master/scoresaber/score_saber_data_v2.json";
 
         public static ScoreSaberDatabaseDownloader Instance;