SongDataCore migration (#86)

* Migrate to SongDataCore.
* Sort buttons disable if the data they need is not available.
* Speed up large playlist matching by mapping the songs first.
* Update manifest with dependencies.
* Update version.

SongBrowserPlugin/DataAccess/Network/CacheableDownloadHandler.cs

@@ -1,204 +0,0 @@
-using UnityEngine.Networking;
-using System.IO;
-using System.Security.Cryptography;
-using System.Text;
-using System;
-using UnityEngine;
-
-using Logger = SongBrowser.Logging.Logger;
-
-// Modified Version of:
-// https://github.com/mob-sakai/AssetSystem/blob/master/Assets/Mobcast/Coffee/AssetSystem/CacheableDownloadHandler.cs
-// MIT-LICENSE - https://github.com/mob-sakai/AssetSystem/blob/master/LICENSE
-// Modified to use custom logging only.
-namespace Mobcast.Coffee.AssetSystem
-{
-
-	public static class UnityWebRequestCachingExtensions
-	{
-		/// <summary>
-		/// Set UnityWebRequest to be cacheable(Etag).
-		/// </summary>
-		public static void SetCacheable(this UnityWebRequest www, CacheableDownloadHandler handler)
-		{
-			var etag = CacheableDownloadHandler.GetCacheEtag(www.url);
-			if (etag != null)
-				www.SetRequestHeader("If-None-Match", etag);
-			www.downloadHandler = handler;
-		}
-	}
-
-	/// <summary>
-	/// Cacheable download handler texture.
-	/// </summary>
-	public class CacheableDownloadHandlerTexture : CacheableDownloadHandler
-	{
-		Texture2D m_Texture;
-
-		public CacheableDownloadHandlerTexture(UnityWebRequest www, byte[] preallocateBuffer)
-			: base(www, preallocateBuffer)
-		{
-		}
-
-		/// <summary>
-		/// Returns the downloaded Texture, or null.
-		/// </summary>
-		public Texture2D Texture
-		{
-			get
-			{
-				if (m_Texture == null)
-				{
-					m_Texture = new Texture2D(1, 1);
-					m_Texture.LoadImage(GetData(), true);
-				}
-				return m_Texture;
-			}
-		}
-	}
-
-	/// <summary>
-	/// Cacheable download handler.
-	/// </summary>
-	public abstract class CacheableDownloadHandler : DownloadHandlerScript
-	{
-        const string kLog = "[WebRequestCaching] ";
-		const string kDataSufix = "_d";
-		const string kEtagSufix = "_e";
-
-		static string s_WebCachePath; 
-		static SHA1CryptoServiceProvider s_SHA1 = new SHA1CryptoServiceProvider();
-
-		/// <summary>
-		/// Is the download already finished?
-		/// </summary>
-		public new bool isDone { get; private set; }
-
-
-		UnityWebRequest m_WebRequest;
-		MemoryStream m_Stream;
-		protected byte[] m_Buffer;
-
-		internal CacheableDownloadHandler(UnityWebRequest www, byte[] preallocateBuffer)
-			: base(preallocateBuffer)
-		{
-			this.m_WebRequest = www;
-			m_Stream = new MemoryStream(preallocateBuffer.Length);
-		}
-
-		/// <summary>
-		/// Get path for web-caching.
-		/// </summary>
-		public static string GetCachePath(string url)
-		{
-			if (s_WebCachePath == null)
-			{
-				s_WebCachePath = Application.temporaryCachePath + "/WebCache/";
-                Logger.Debug("{0}WebCachePath : {1}", kLog, s_WebCachePath);
-
-			}
-
-			if (!Directory.Exists(s_WebCachePath))
-				Directory.CreateDirectory(s_WebCachePath);
-
-			return s_WebCachePath + Convert.ToBase64String(s_SHA1.ComputeHash(UTF8Encoding.Default.GetBytes(url))).Replace('/', '_');
-		}
-
-		/// <summary>
-		/// Get cached Etag for url.
-		/// </summary>
-		public static string GetCacheEtag(string url)
-		{
-			var path = GetCachePath(url);
-			var infoPath = path + kEtagSufix;
-			var dataPath = path + kDataSufix;
-			return File.Exists(infoPath) && File.Exists(dataPath)
-					? File.ReadAllText(infoPath)
-					: null;
-		}
-
-		/// <summary>
-		/// Load cached data for url.
-		/// </summary>
-		public static byte[] LoadCache(string url)
-		{
-			return File.ReadAllBytes(GetCachePath(url) + kDataSufix);
-		}
-
-		/// <summary>
-		/// Save cache data for url.
-		/// </summary>
-		public static void SaveCache(string url, string etag, byte[] datas)
-		{
-			var path = GetCachePath(url);
-			File.WriteAllText(path + kEtagSufix, etag);
-			File.WriteAllBytes(path + kDataSufix, datas);
-		}
-
-		/// <summary>
-		/// Callback, invoked when the data property is accessed.
-		/// </summary>
-		protected override byte[] GetData()
-		{
-			if (!isDone)
-			{
-				Logger.Error("{0}Downloading is not completed : {1}", kLog, m_WebRequest.url);
-				throw new InvalidOperationException("Downloading is not completed. " + m_WebRequest.url);
-			}
-			else if (m_Buffer == null)
-			{
-				// Etag cache hit!
-				if (m_WebRequest.responseCode == 304)
-				{
-					Logger.Debug("<color=green>{0}Etag cache hit : {1}</color>", kLog, m_WebRequest.url);
-					m_Buffer = LoadCache(m_WebRequest.url);
-				}
-				// Download is completed successfully.
-				else if (m_WebRequest.responseCode == 200)
-				{
-					Logger.Debug("<color=green>{0}Download is completed successfully : {1}</color>", kLog, m_WebRequest.url);
-					m_Buffer = m_Stream.GetBuffer();
-					SaveCache(m_WebRequest.url, m_WebRequest.GetResponseHeader("Etag"), m_Buffer);
-				}
-			}
-
-			if (m_Stream != null)
-			{
-				m_Stream.Dispose();
-				m_Stream = null;
-			}
-			return m_Buffer;
-		}
-
-		/// <summary>
-		/// Callback, invoked as data is received from the remote server.
-		/// </summary>
-		protected override bool ReceiveData(byte[] data, int dataLength)
-		{
-			m_Stream.Write(data, 0, dataLength);
-			return true;
-		}
-
-		/// <summary>
-		/// Callback, invoked when all data has been received from the remote server.
-		/// </summary>
-		protected override void CompleteContent()
-		{
-			base.CompleteContent();
-			isDone = true;
-		}
-
-		/// <summary>
-		/// Signals that this [DownloadHandler] is no longer being used, and should clean up any resources it is using.
-		/// </summary>
-		public new void Dispose()
-		{
-			base.Dispose();
-			if (m_Stream != null)
-			{
-				m_Stream.Dispose();
-				m_Stream = null;
-			}
-		}
-	}
-}

SongBrowserPlugin/DataAccess/Network/CacheableDownloadHandlerScoreSaberData.cs

@@ -1,33 +0,0 @@
-using Mobcast.Coffee.AssetSystem;
-using UnityEngine.Networking;
-
-namespace SongBrowser.DataAccess.Network
-{
-    /// <summary>
-	/// Cacheable download handler for score saber tsv file.
-	/// </summary>
-	public class CacheableDownloadHandlerScoreSaberData : CacheableDownloadHandler
-    {
-        ScoreSaberDataFile _scoreSaberDataFile;
-
-        public CacheableDownloadHandlerScoreSaberData(UnityWebRequest www, byte[] preallocateBuffer)
-            : base(www, preallocateBuffer)
-        {
-        }
-
-        /// <summary>
-        /// Returns the downloaded score saber data file, or null.
-        /// </summary>
-        public ScoreSaberDataFile ScoreSaberDataFile
-        {
-            get
-            {
-                if (_scoreSaberDataFile == null)
-                {
-                    _scoreSaberDataFile = new ScoreSaberDataFile(GetData());
-                }
-                return _scoreSaberDataFile;
-            }
-        }
-    }
-}

SongBrowserPlugin/DataAccess/Playlist.cs

@@ -141,9 +141,20 @@ namespace SongBrowser.DataAccess
 
         public static void MatchSongsForPlaylist(Playlist playlist, bool matchAll = false)
         {
-            //bananbread playlist id  
             if (!SongCore.Loader.AreSongsLoaded || SongCore.Loader.AreSongsLoading || playlist.playlistTitle == "All songs" || playlist.playlistTitle == "Your favorite songs") return;
 
+            Dictionary<string, CustomPreviewBeatmapLevel> songMap = new Dictionary<string, CustomPreviewBeatmapLevel>(StringComparer.OrdinalIgnoreCase);
+            foreach (var kp in SongCore.Loader.CustomLevels)
+            {
+                var songHash = CustomHelpers.GetSongHash(kp.Value.levelID);
+                if (songMap.ContainsKey(songHash))
+                {
+                    continue;
+                }
+
+                songMap.Add(songHash, kp.Value);
+            }
+
             if (!playlist.songs.All(x => x.level != null) || matchAll)
             {
                 playlist.songs.AsParallel().ForAll(x =>
@@ -152,22 +163,35 @@ namespace SongBrowser.DataAccess
                     {
                         try
                         {
-                            if (!string.IsNullOrEmpty(x.levelId)) //check that we have levelId and if we do, try to match level
+                            // try to use levelID
+                            if (!string.IsNullOrEmpty(x.levelId))
                             {
-                                x.level = SongCore.Loader.CustomLevels.Values.FirstOrDefault(y => y.levelID == x.levelId);
+                                string songHash = CustomHelpers.GetSongHash(x.levelId);
+                                if (songMap.ContainsKey(songHash))
+                                {
+                                    x.level = songMap[songHash];
+                                    x.hash = songHash;
+                                }
                             }
-                            if (x.level == null && !string.IsNullOrEmpty(x.hash)) //if level is still null, check that we have hash and if we do, try to match level
+
+                            // failed, try again using hash
+                            if (x.level == null && !string.IsNullOrEmpty(x.hash))
                             {
+                                // fix broken playlists from a bug in song browser
                                 if (x.hash.Contains("custom_level"))
                                 {
                                     x.hash = CustomHelpers.GetSongHash(x.hash);
                                 }
-                                x.level = SongCore.Loader.CustomLevels.Values.FirstOrDefault(y => string.Equals(CustomHelpers.GetSongHash(y.levelID), x.hash, StringComparison.OrdinalIgnoreCase));
+
+                                if (songMap.ContainsKey(x.hash))
+                                {
+                                    x.level = songMap[x.hash];
+                                }
                             }
+
                             if (x.level == null && !string.IsNullOrEmpty(x.key))
                             {
                                 x.level = SongCore.Loader.CustomLevels.FirstOrDefault(y => y.Value.customLevelPath.Contains(x.key)).Value;
-
                                 if (x.level != null && !String.IsNullOrEmpty(x.level.levelID))
                                 {
                                     x.hash = CustomHelpers.GetSongHash(x.level.levelID);
@@ -217,29 +241,25 @@ namespace SongBrowser.DataAccess
             if (!string.IsNullOrEmpty(key) || level == null || !(level is CustomPreviewBeatmapLevel))
                 yield break;
 
+            string songHash = null;
             if (!string.IsNullOrEmpty(hash))
             {
-                ScrappedSong song = ScrappedData.Songs.FirstOrDefault(x => hash.ToUpper() == x.Hash);
-                if (song != null)
-                    key = song.Key;
-                else
-                    yield return SongDownloader.Instance.RequestSongByLevelIDCoroutine(hash, (Song bsSong) => { if (bsSong != null) key = bsSong.key; });
+                songHash = hash;
             }
             else if (!string.IsNullOrEmpty(levelId))
             {
-                ScrappedSong song = ScrappedData.Songs.FirstOrDefault(x => levelId.StartsWith(x.Hash));
-                if (song != null)
-                    key = song.Key;
-                else
-                    yield return SongDownloader.Instance.RequestSongByLevelIDCoroutine(level.levelID.Split('_')[2], (Song bsSong) => { if (bsSong != null) key = bsSong.key; });
+                songHash = CustomHelpers.GetSongHash(level.levelID);
             }
-            else if (level != null)
+            
+            if (songHash != null && SongDataCore.Plugin.BeatSaver.Data.Songs.ContainsKey(hash))
             {
-                ScrappedSong song = ScrappedData.Songs.FirstOrDefault(x => level.levelID.StartsWith(x.Hash));
-                if (song != null)
-                    key = song.Key;
-                else
-                    yield return SongDownloader.Instance.RequestSongByLevelIDCoroutine(level.levelID.Split('_')[2], (Song bsSong) => { if (bsSong != null) key = bsSong.key; });
+                var song = SongDataCore.Plugin.BeatSaver.Data.Songs[hash];
+                key = song.key;
+            }
+            else
+            {
+                // no more hitting api just to match a key.  We know the song hash.
+                //yield return SongDownloader.Instance.RequestSongByLevelIDCoroutine(level.levelID.Split('_')[2], (Song bsSong) => { if (bsSong != null) key = bsSong.key; });
             }
         }
     }
@@ -329,9 +349,8 @@ namespace SongBrowser.DataAccess
         }
 
         public void SavePlaylist(string path = "")
-        {
-            if (ScrappedData.Songs.Count > 0)
-                SharedCoroutineStarter.instance.StartCoroutine(SavePlaylistCoroutine(path));
+        {            
+            SharedCoroutineStarter.instance.StartCoroutine(SavePlaylistCoroutine(path));
         }
 
         public IEnumerator SavePlaylistCoroutine(string path = "")
@@ -354,9 +373,14 @@ namespace SongBrowser.DataAccess
                 Logger.Exception("Unable to save playlist! Exception: " + e);
                 yield break;
             }
-            foreach (PlaylistSong song in songs)
+
+            // match key if we can, not really that important anymore
+            if (SongDataCore.Plugin.BeatSaver.Data.Songs.Count > 0)
             {
-                yield return song.MatchKey();
+                foreach (PlaylistSong song in songs)
+                {
+                    yield return song.MatchKey();
+                }
             }
 
             try

SongBrowserPlugin/DataAccess/ScoreSaberDatabase.cs

@@ -1,43 +0,0 @@
-using Newtonsoft.Json;
-using System;
-using System.Collections.Generic;
-using System.Diagnostics;
-using Logger = SongBrowser.Logging.Logger;
-
-namespace SongBrowser.DataAccess
-{
-    public class ScoreSaberSong
-    {
-        public string song { get; set; }
-        public string mapper { get; set; }
-        public List<ScoreSaberSongDifficultyStats> diffs { get; set; }
-    }
-
-    public class ScoreSaberSongDifficultyStats
-    {
-        public string diff { get; set; }
-        public long scores { get; set; }
-        public double star { get; set; }
-        public double pp { get; set; }
-    }
-
-    public class ScoreSaberDataFile
-    {
-        public Dictionary<String, ScoreSaberSong> SongHashToScoreSaberData = null;
-
-        public ScoreSaberDataFile(byte[] data)
-        {
-            Stopwatch timer = new Stopwatch();
-            timer.Start();
-
-            System.Globalization.NumberStyles style = System.Globalization.NumberStyles.AllowDecimalPoint;
-
-            string result = System.Text.Encoding.UTF8.GetString(data);
-
-            SongHashToScoreSaberData = JsonConvert.DeserializeObject<Dictionary<string, ScoreSaberSong>>(result);
-                        
-            timer.Stop();
-            Logger.Debug("Processing ScoreSaber data took {0}ms", timer.ElapsedMilliseconds);
-        }
-    }
-}

SongBrowserPlugin/DataAccess/ScrappedData.cs

@@ -1,110 +0,0 @@
-using Newtonsoft.Json;
-using System;
-using System.Collections;
-using System.Collections.Generic;
-using System.Linq;
-using UnityEngine;
-using UnityEngine.Networking;
-using Logger = SongBrowser.Logging.Logger;
-
-namespace SongBrowser.DataAccess
-{
-    public class ScrappedSong
-    {
-        public string Key { get; set; }
-        public string Hash { get; set; }
-        public string SongName { get; set; }
-        public string SongSubName { get; set; }
-        public string LevelAuthorName { get; set; }
-        public string SongAuthorName { get; set; }
-        public List<DifficultyStats> Diffs { get; set; }
-        public long Bpm { get; set; }
-        public long PlayedCount { get; set; }
-        public long Upvotes { get; set; }
-        public long Downvotes { get; set; }
-        public float Heat { get; set; }
-        public float Rating { get; set; }
-    }
-
-    public class DifficultyStats
-    {
-        public string Diff { get; set; }
-        public long Scores { get; set; }
-        public double Stars { get; set; }
-    }
-
-    public class ScrappedData : MonoBehaviour
-    {
-        private static ScrappedData _instance = null;
-        public static ScrappedData Instance
-        {
-            get
-            {
-                if (!_instance)
-                {
-                    _instance = new GameObject("ScrappedData").AddComponent<ScrappedData>();
-                    DontDestroyOnLoad(_instance.gameObject);
-                }
-                return _instance;
-            }
-            private set
-            {
-                _instance = value;
-            }
-        }
-
-        public static List<ScrappedSong> Songs = new List<ScrappedSong>();
-
-        public static string scrappedDataURL = "https://raw.githubusercontent.com/andruzzzhka/BeatSaberScrappedData/master/combinedScrappedData.json";
-
-        public IEnumerator DownloadScrappedData(Action<List<ScrappedSong>> callback)
-        {
-            Logger.Info("Downloading scrapped data...");
-
-            UnityWebRequest www;
-            bool timeout = false;
-            float time = 0f;
-            UnityWebRequestAsyncOperation asyncRequest;
-
-            try
-            {
-                www = UnityWebRequest.Get(scrappedDataURL);
-
-                asyncRequest = www.SendWebRequest();
-            }
-            catch (Exception e)
-            {
-                Logger.Error(e);
-                yield break;
-            }
-
-            while (!asyncRequest.isDone)
-            {
-                yield return null;
-                time += Time.deltaTime;
-                if (time >= 5f && asyncRequest.progress <= float.Epsilon)
-                {
-                    www.Abort();
-                    timeout = true;
-                    Logger.Error("Connection timed out!");
-                }
-            }
-
-
-            if (www.isNetworkError || www.isHttpError || timeout)
-            {
-                Logger.Error("Unable to download scrapped data! " + (www.isNetworkError ? $"Network error: {www.error}" : (www.isHttpError ? $"HTTP error: {www.error}" : "Unknown error")));
-            }
-            else
-            {
-                Logger.Info("Received response from github.com...");
-
-                Songs = JsonConvert.DeserializeObject<List<ScrappedSong>>(www.downloadHandler.text).OrderByDescending(x => x.Diffs.Count > 0 ? x.Diffs.Max(y => y.Stars) : 0).ToList();
-
-                callback?.Invoke(Songs);
-                Logger.Info("Scrapped data downloaded!");
-            }
-        }
-
-    }
-}

SongBrowserPlugin/DataAccess/SongBrowserModel.cs

@@ -30,7 +30,6 @@ namespace SongBrowser
         private Dictionary<String, double> _cachedLastWriteTimes;
         private Dictionary<string, int> _weights;
         private Dictionary<BeatmapDifficulty, int> _difficultyWeights;
-        private Dictionary<string, ScrappedSong> _levelHashToDownloaderData = null;
         private Dictionary<string, int> _levelIdToPlayCount;
 
         public BeatmapCharacteristicSO CurrentBeatmapCharacteristicSO;
@@ -339,24 +338,6 @@ namespace SongBrowser
         }
         
         /// <summary>
-        /// Map the downloader data for quick lookup.
-        /// </summary>
-        /// <param name="songs"></param>
-        public void UpdateDownloaderDataMapping(List<ScrappedSong> songs)
-        {
-            _levelHashToDownloaderData = new Dictionary<string, ScrappedSong>();
-            foreach (ScrappedSong song in songs)
-            {
-                if (_levelHashToDownloaderData.ContainsKey(song.Hash))
-                {
-                    continue;
-                }
-
-                _levelHashToDownloaderData.Add(song.Hash, song);
-            }
-        }
-
-        /// <summary>
         /// Add Song to Editing Playlist
         /// </summary>
         /// <param name="songInfo"></param>
@@ -676,9 +657,9 @@ namespace SongBrowser
             {
                 var hash = CustomHelpers.GetSongHash(x.levelID);
                 double maxPP = 0.0;
-                if (ScoreSaberDatabaseDownloader.ScoreSaberDataFile.SongHashToScoreSaberData.ContainsKey(hash))
+                if (SongDataCore.Plugin.ScoreSaber.Data.Songs.ContainsKey(hash))
                 {
-                     maxPP = ScoreSaberDatabaseDownloader.ScoreSaberDataFile.SongHashToScoreSaberData[hash].diffs.Max(y => y.pp);
+                     maxPP = SongDataCore.Plugin.ScoreSaber.Data.Songs[hash].diffs.Max(y => y.pp);
                 }
 
                 if (maxPP > 0f)
@@ -754,7 +735,7 @@ namespace SongBrowser
         {
             Logger.Info("Sorting song list by performance points...");
 
-            if (ScoreSaberDatabaseDownloader.ScoreSaberDataFile == null)
+            if (!SongDataCore.Plugin.ScoreSaber.IsDataAvailable())
             {
                 return levels;
             }
@@ -763,9 +744,9 @@ namespace SongBrowser
                 .OrderByDescending(x =>
                 {
                     var hash = CustomHelpers.GetSongHash(x.levelID);
-                    if (ScoreSaberDatabaseDownloader.ScoreSaberDataFile.SongHashToScoreSaberData.ContainsKey(hash))
+                    if (SongDataCore.Plugin.ScoreSaber.Data.Songs.ContainsKey(hash))
                     {
-                        return ScoreSaberDatabaseDownloader.ScoreSaberDataFile.SongHashToScoreSaberData[hash].diffs.Max(y => y.pp);
+                        return SongDataCore.Plugin.ScoreSaber.Data.Songs[hash].diffs.Max(y => y.pp);
                     }
                     else
                     {
@@ -784,7 +765,7 @@ namespace SongBrowser
         {
             Logger.Info("Sorting song list by star points...");
 
-            if (ScoreSaberDatabaseDownloader.ScoreSaberDataFile == null)
+            if (!SongDataCore.Plugin.ScoreSaber.IsDataAvailable())
             {
                 return levels;
             }
@@ -794,9 +775,9 @@ namespace SongBrowser
                 {
                     var hash = CustomHelpers.GetSongHash(x.levelID);
                     var stars = 0.0;
-                    if (ScoreSaberDatabaseDownloader.ScoreSaberDataFile.SongHashToScoreSaberData.ContainsKey(hash))
+                    if (SongDataCore.Plugin.ScoreSaber.Data.Songs.ContainsKey(hash))
                     {
-                        var diffs = ScoreSaberDatabaseDownloader.ScoreSaberDataFile.SongHashToScoreSaberData[hash].diffs;   
+                        var diffs = SongDataCore.Plugin.ScoreSaber.Data.Songs[hash].diffs;   
                         stars = diffs.Max(y => y.star);
                     }
 
@@ -903,7 +884,7 @@ namespace SongBrowser
             Logger.Info("Sorting song list by BeatSaver UpVotes");
 
             // Do not always have data when trying to sort by UpVotes
-            if (_levelHashToDownloaderData == null)
+            if (!SongDataCore.Plugin.BeatSaver.IsDataAvailable())
             {
                 return levelIds;
             }
@@ -911,9 +892,9 @@ namespace SongBrowser
             return levelIds
                 .OrderByDescending(x => {
                     var hash = CustomHelpers.GetSongHash(x.levelID);
-                    if (_levelHashToDownloaderData.ContainsKey(hash))
+                    if (SongDataCore.Plugin.BeatSaver.Data.Songs.ContainsKey(hash))
                     {
-                        return _levelHashToDownloaderData[hash].Upvotes;
+                        return SongDataCore.Plugin.BeatSaver.Data.Songs[hash].stats.upVotes;
                     }
                     else
                     {
@@ -933,7 +914,7 @@ namespace SongBrowser
             Logger.Info("Sorting song list by BeatSaver PlayCount");
 
             // Do not always have data when trying to sort by UpVotes
-            if (_levelHashToDownloaderData == null)
+            if (!SongDataCore.Plugin.BeatSaver.IsDataAvailable())
             {
                 return levelIds;
             }
@@ -941,9 +922,9 @@ namespace SongBrowser
             return levelIds
                 .OrderByDescending(x => {
                     var hash = CustomHelpers.GetSongHash(x.levelID);
-                    if (_levelHashToDownloaderData.ContainsKey(hash))
+                    if (SongDataCore.Plugin.BeatSaver.Data.Songs.ContainsKey(hash))
                     {
-                        return _levelHashToDownloaderData[hash].PlayedCount;
+                        return SongDataCore.Plugin.BeatSaver.Data.Songs[hash].stats.plays;
                     }
                     else
                     {
@@ -963,7 +944,7 @@ namespace SongBrowser
             Logger.Info("Sorting song list by BeatSaver Rating!");
 
             // Do not always have data when trying to sort by rating
-            if (_levelHashToDownloaderData == null)
+            if (!SongDataCore.Plugin.BeatSaver.IsDataAvailable())
             {
                 return levelIds;
             }
@@ -971,9 +952,9 @@ namespace SongBrowser
             return levelIds
                 .OrderByDescending(x => {
                     var hash = CustomHelpers.GetSongHash(x.levelID);
-                    if (_levelHashToDownloaderData.ContainsKey(hash))
+                    if (SongDataCore.Plugin.BeatSaver.Data.Songs.ContainsKey(hash))
                     {
-                        return _levelHashToDownloaderData[hash].Rating;
+                        return SongDataCore.Plugin.BeatSaver.Data.Songs[hash].stats.rating;
                     }
                     else
                     {
@@ -993,7 +974,7 @@ namespace SongBrowser
             Logger.Info("Sorting song list by BeatSaver Heat!");
 
             // Do not always have data when trying to sort by heat
-            if (_levelHashToDownloaderData == null)
+            if (!SongDataCore.Plugin.BeatSaver.IsDataAvailable())
             {
                 return levelIds;
             }
@@ -1001,9 +982,9 @@ namespace SongBrowser
             return levelIds
                 .OrderByDescending(x => {
                     var hash = CustomHelpers.GetSongHash(x.levelID);
-                    if (_levelHashToDownloaderData.ContainsKey(hash))
+                    if (SongDataCore.Plugin.BeatSaver.Data.Songs.ContainsKey(hash))
                     {
-                        return _levelHashToDownloaderData[hash].Heat;
+                        return SongDataCore.Plugin.BeatSaver.Data.Songs[hash].stats.heat;
                     }
                     else
                     {

SongBrowserPlugin/DataAccess/SongBrowserSettings.cs

@@ -34,6 +34,35 @@ namespace SongBrowser.DataAccess
         Search
     }
 
+    static class SongSortModeMethods
+    {
+        public static bool NeedsBeatSaverData(this SongSortMode s)
+        {
+            switch (s)
+            {
+                case SongSortMode.UpVotes:
+                case SongSortMode.Rating:
+                case SongSortMode.PlayCount:
+                case SongSortMode.Heat:
+                    return true;
+                default:
+                    return false;
+            }
+        }
+
+        public static bool NeedsScoreSaberData(this SongSortMode s)
+        {
+            switch (s)
+            {
+                case SongSortMode.PP:
+                case SongSortMode.Stars:
+                    return true;
+                default:
+                    return false;
+            }
+        }
+    }
+
     [Serializable]
     public enum SongFilterMode
     {

SongBrowserPlugin/Plugin.cs

@@ -12,7 +12,7 @@ namespace SongBrowser
 {
     public class Plugin : IBeatSaberPlugin
     {
-        public const string VERSION_NUMBER = "5.2.3";
+        public const string VERSION_NUMBER = "5.3.0";
         public static Plugin Instance;
         public static IPA.Logging.Logger Log;
 

SongBrowserPlugin/Properties/AssemblyInfo.cs

@@ -5,7 +5,7 @@ using System.Runtime.InteropServices;
 // set of attributes. Change these attribute values to modify the information
 // associated with an assembly.
 [assembly: AssemblyTitle("SongBrowser")]
-[assembly: AssemblyDescription("")]
+[assembly: AssemblyDescription("BeatSaber Plugin")]
 [assembly: AssemblyConfiguration("")]
 [assembly: AssemblyCompany("")]
 [assembly: AssemblyProduct("SongBrowser")]
@@ -31,5 +31,5 @@ using System.Runtime.InteropServices;
 // You can specify all the values or you can default the Build and Revision Numbers 
 // by using the '*' as shown below:
 // [assembly: AssemblyVersion("1.0.*")]
-[assembly: AssemblyVersion("5.2.3")]
-[assembly: AssemblyFileVersion("5.2.3")]
+[assembly: AssemblyVersion("5.3.0")]
+[assembly: AssemblyFileVersion("5.3.0")]

SongBrowserPlugin/SongBrowser.csproj

@@ -69,6 +69,9 @@
     <Reference Include="SongCore">
       <HintPath>D:\Games\Steam\SteamApps\common\Beat Saber\Plugins\SongCore.dll</HintPath>
     </Reference>
+    <Reference Include="SongDataCore">
+      <HintPath>D:\Games\Steam\SteamApps\common\Beat Saber\Plugins\SongDataCore.dll</HintPath>
+    </Reference>
     <Reference Include="System" />
     <Reference Include="System.Data" />
     <Reference Include="System.IO.Compression" />
@@ -128,12 +131,8 @@
     <Compile Include="DataAccess\BeatSaverApi\BeatSaverApiResults.cs" />
     <Compile Include="DataAccess\FileOperationApiWrapper.cs" />
     <Compile Include="DataAccess\LoadScripts.cs" />
-    <Compile Include="DataAccess\Network\CacheableDownloadHandler.cs" />
-    <Compile Include="DataAccess\Network\CacheableDownloadHandlerScoreSaberData.cs" />
-    <Compile Include="DataAccess\Playlist.cs" />
-    <Compile Include="DataAccess\ScoreSaberDatabase.cs" />
+    <Compile Include="DataAccess\Playlist.cs" />
     <Compile Include="DataAccess\Network\Downloader.cs" />
-    <Compile Include="DataAccess\ScrappedData.cs" />
     <Compile Include="Internals\BSEvents.cs" />
     <Compile Include="Internals\CustomHelpers.cs" />
     <Compile Include="Internals\SimpleJSON.cs" />
@@ -143,7 +142,6 @@
     <Compile Include="UI\DownloadQueue\DownloadQueueTableCell.cs" />
     <Compile Include="UI\DownloadQueue\DownloadQueueViewController.cs" />
     <Compile Include="UI\ProgressBar.cs" />
-    <Compile Include="UI\ScoreSaberDatabaseDownloader.cs" />
     <Compile Include="SongBrowserApplication.cs" />
     <Compile Include="Plugin.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
@@ -193,4 +191,4 @@ IF EXIST "C:\Program Files (x86)\Steam\steamapps\common\Beat Saber\Plugins" xcop
     <Target Name="AfterBuild">
     </Target>
     -->
-</Project>
+</Project>

SongBrowserPlugin/SongBrowserApplication.cs

@@ -16,7 +16,6 @@ namespace SongBrowser
 
         // Song Browser UI Elements
         private SongBrowserUI _songBrowserUI;
-        private ScoreSaberDatabaseDownloader _ppDownloader;
 
         public static SongBrowser.UI.ProgressBar MainProgressBar;
 
@@ -48,8 +47,6 @@ namespace SongBrowser
             Instance = this;
 
             _songBrowserUI = gameObject.AddComponent<SongBrowserUI>();
-            _ppDownloader = gameObject.AddComponent<ScoreSaberDatabaseDownloader>();
-            _ppDownloader.onScoreSaberDataDownloaded += OnScoreSaberDataDownloaded;
         }
 
         /// <summary>
@@ -61,8 +58,8 @@ namespace SongBrowser
 
             InstallHandlers();
 
-            // Initialize Downloader Scrapped Data
-            StartCoroutine(ScrappedData.Instance.DownloadScrappedData(OnDownloaderScrappedDataDownloaded));
+            SongDataCore.Plugin.ScoreSaber.OnDataFinishedProcessing += OnScoreSaberDataDownloaded;
+            SongDataCore.Plugin.BeatSaver.OnDataFinishedProcessing += OnBeatSaverDataDownloaded;
 
             if (SongCore.Loader.AreSongsLoaded)
             {
@@ -114,10 +111,14 @@ namespace SongBrowser
             Logger.Trace("OnScoreSaberDataDownloaded");
             try
             {
-                if (_songBrowserUI.Model.Settings.sortMode == SongSortMode.PP)
+                if (_songBrowserUI.Model.Settings.sortMode.NeedsScoreSaberData())
                 {
                     _songBrowserUI.ProcessSongList();
-                    _songBrowserUI.RefreshSongList();
+                    _songBrowserUI.RefreshSongUI();
+                }
+                else
+                {
+                    _songBrowserUI.RefreshSortButtonUI();
                 }
             }
             catch (Exception e)
@@ -129,17 +130,19 @@ namespace SongBrowser
         /// <summary>
         /// Update mapping of scrapped song data.
         /// </summary>
-        private void OnDownloaderScrappedDataDownloaded(List<ScrappedSong> songs)
+        private void OnBeatSaverDataDownloaded()
         {
-            Logger.Trace("OnDownloaderScrappedDataDownloaded");
+            Logger.Trace("OnBeatSaverDataDownloaded");
             try
             {
-                PlaylistsCollection.MatchSongsForAllPlaylists(true);
-                _songBrowserUI.Model.UpdateDownloaderDataMapping(songs);
-                if (_songBrowserUI.Model.Settings.sortMode == SongSortMode.UpVotes)
+                if (_songBrowserUI.Model.Settings.sortMode.NeedsBeatSaverData())
                 {
                     _songBrowserUI.ProcessSongList();
-                    _songBrowserUI.RefreshSongList();
+                    _songBrowserUI.RefreshSongUI();
+                }
+                else
+                {
+                    _songBrowserUI.RefreshSortButtonUI();
                 }
             }
             catch (Exception e)

SongBrowserPlugin/UI/Browser/SongBrowserUI.cs

@@ -13,6 +13,7 @@ using System.Collections;
 using SongCore.Utilities;
 using SongBrowser.Internals;
 using CustomUI.BeatSaber;
+using SongDataCore.ScoreSaber;
 
 namespace SongBrowser.UI
 {
@@ -306,7 +307,7 @@ namespace SongBrowser.UI
                     },
                     sortButtonNames[i]);
                 sortButton.Button.SetButtonTextSize(sortButtonFontSize);
-                sortButton.Button.GetComponentsInChildren<HorizontalLayoutGroup>().First(btn => btn.name == "Content").padding = new RectOffset(4, 4, 2, 2);
+                sortButton.Button.GetComponentsInChildren<HorizontalLayoutGroup>().First(btn => btn.name == "Content").padding = new RectOffset(4, 4, 2, 2);                
                 sortButton.Button.ToggleWordWrapping(false);
                 sortButton.Button.name = "Sort" + sortModes[i].ToString() + "Button";
 
@@ -530,6 +531,11 @@ namespace SongBrowser.UI
         /// </summary>
         public void RefreshSongUI(bool scrollToLevel=true)
         {
+            if (!_uiCreated)
+            {
+                return;
+            }
+
             RefreshSongList(scrollToLevel);
             RefreshSortButtonUI();
             if (!scrollToLevel)
@@ -678,6 +684,13 @@ namespace SongBrowser.UI
         {
             Logger.Debug("Sort button - {0} - pressed.", sortMode.ToString());
 
+            if ((sortMode.NeedsScoreSaberData() && !SongDataCore.Plugin.ScoreSaber.IsDataAvailable()) ||
+                (sortMode.NeedsBeatSaverData() && !SongDataCore.Plugin.BeatSaver.IsDataAvailable()))
+            {
+                Logger.Info("Data for sort type is not available.");
+                return;
+            }
+
             // Clear current selected level id so our song list jumps to the start
             _model.LastSelectedLevelId = null;
 
@@ -1115,7 +1128,7 @@ namespace SongBrowser.UI
         {
             Logger.Trace("RefreshScoreSaberData({0})", level.levelID);
 
-            if (ScoreSaberDatabaseDownloader.ScoreSaberDataFile == null)
+            if (!SongDataCore.Plugin.ScoreSaber.IsDataAvailable())
             {
                 return;
             }
@@ -1131,10 +1144,10 @@ namespace SongBrowser.UI
             // Check if we have data for this song
             Logger.Debug("Checking if have info for song {0}", level.songName);
             var hash = CustomHelpers.GetSongHash(level.levelID);
-            if (ScoreSaberDatabaseDownloader.ScoreSaberDataFile.SongHashToScoreSaberData.ContainsKey(hash))
+            if (SongDataCore.Plugin.ScoreSaber.Data.Songs.ContainsKey(hash))
             {
                 Logger.Debug("Checking if have difficulty for song {0} difficulty {1}", level.songName, difficultyString);
-                ScoreSaberSong scoreSaberSong = ScoreSaberDatabaseDownloader.ScoreSaberDataFile.SongHashToScoreSaberData[hash];
+                ScoreSaberSong scoreSaberSong = SongDataCore.Plugin.ScoreSaber.Data.Songs[hash];
                 ScoreSaberSongDifficultyStats scoreSaberSongDifficulty = scoreSaberSong.diffs.FirstOrDefault(x => String.Equals(x.diff, difficultyString));
                 if (scoreSaberSongDifficulty != null)
                 {
@@ -1334,10 +1347,27 @@ namespace SongBrowser.UI
         /// </summary>
         public void RefreshSortButtonUI()
         {
+            if (!_uiCreated)
+            {
+                return;
+            }
+
             // So far all we need to refresh is the sort buttons.
             foreach (SongSortButton sortButton in _sortButtonGroup)
             {
-                UIBuilder.SetButtonBorder(sortButton.Button, Color.white);
+                if (sortButton.SortMode.NeedsBeatSaverData() && !SongDataCore.Plugin.BeatSaver.IsDataAvailable())
+                {
+                    UIBuilder.SetButtonBorder(sortButton.Button, Color.gray);
+                }
+                else if (sortButton.SortMode.NeedsScoreSaberData() && !SongDataCore.Plugin.ScoreSaber.IsDataAvailable())
+                {
+                    UIBuilder.SetButtonBorder(sortButton.Button, Color.gray);
+                }
+                else
+                {
+                    UIBuilder.SetButtonBorder(sortButton.Button, Color.white);
+                }
+
                 if (sortButton.SortMode == _model.Settings.sortMode)
                 {
                     if (this._model.Settings.invertSortResults)

SongBrowserPlugin/UI/ScoreSaberDatabaseDownloader.cs

@@ -1,122 +0,0 @@
-using Mobcast.Coffee.AssetSystem;
-using SongBrowser.DataAccess;
-using SongBrowser.DataAccess.Network;
-using System;
-using System.Collections;
-using UnityEngine;
-using UnityEngine.Networking;
-using Logger = SongBrowser.Logging.Logger;
-
-namespace SongBrowser.UI
-{
-    public class ScoreSaberDatabaseDownloader : MonoBehaviour
-    {
-        public const String SCRAPED_SCORE_SABER_ALL_JSON_URL = "https://cdn.wes.cloud/beatstar/bssb/v2-all.json";
-        public const String SCRAPED_SCORE_SABER_RANKED_JSON_URL = "https://cdn.wes.cloud/beatstar/bssb/v2-ranked.json";
-
-        public static ScoreSaberDatabaseDownloader Instance;
-
-        public static ScoreSaberDataFile ScoreSaberDataFile = null;        
-
-        public Action onScoreSaberDataDownloaded;
-
-        private readonly byte[] _buffer = new byte[4 * 1048576];
-
-        /// <summary>
-        /// Awake.
-        /// </summary>
-        private void Awake()
-        {
-            Logger.Trace("Awake-ScoreSaberDatabaseDownloader()");
-
-            if (Instance == null)
-            {
-                Instance = this;
-            }
-        }
-
-        /// <summary>
-        /// Acquire any UI elements from Beat saber that we need.  Wait for the song list to be loaded.
-        /// </summary>
-        public void Start()
-        {
-            Logger.Trace("Start()");
-
-            StartCoroutine(DownloadScoreSaberDatabases());
-        }
-
-        /// <summary>
-        /// Helper to download both databases.
-        /// </summary>
-        /// <returns></returns>
-        private IEnumerator DownloadScoreSaberDatabases()
-        {
-            ScoreSaberDataFile = null;
-
-            yield return DownloadScoreSaberData(SCRAPED_SCORE_SABER_ALL_JSON_URL);
-            yield return DownloadScoreSaberData(SCRAPED_SCORE_SABER_RANKED_JSON_URL);
-
-            if (ScoreSaberDataFile != null)
-            {
-                SongBrowserApplication.MainProgressBar.ShowMessage("Success downloading BeatStar data...", 5.0f);
-                onScoreSaberDataDownloaded?.Invoke();
-            }
-        }
-
-        /// <summary>
-        /// Wait for score saber related files to download.
-        /// </summary>
-        /// <returns></returns>
-        private IEnumerator DownloadScoreSaberData(String url)
-        {
-            SongBrowserApplication.MainProgressBar.ShowMessage("Downloading BeatStar data...", 5.0f);
-
-            Logger.Info("Attempting to download: {0}", url);
-            using (UnityWebRequest www = UnityWebRequest.Get(url))
-            {
-                // Use 4MB cache, large enough for this file to grow for awhile.
-                www.SetCacheable(new CacheableDownloadHandlerScoreSaberData(www, _buffer));
-                yield return www.SendWebRequest();
-
-                Logger.Debug("Returned from web request!...");
-
-                try
-                {
-                    // First time 
-                    if (ScoreSaberDatabaseDownloader.ScoreSaberDataFile == null)
-                    {
-                        ScoreSaberDatabaseDownloader.ScoreSaberDataFile = (www.downloadHandler as CacheableDownloadHandlerScoreSaberData).ScoreSaberDataFile;                            
-                    }
-                    else
-                    {
-                        // Second time, update.
-                        var newScoreSaberData = (www.downloadHandler as CacheableDownloadHandlerScoreSaberData).ScoreSaberDataFile;
-                        foreach (var pair in newScoreSaberData.SongHashToScoreSaberData)
-                        {
-                            if (ScoreSaberDatabaseDownloader.ScoreSaberDataFile.SongHashToScoreSaberData.ContainsKey(pair.Key))
-                            {
-                                foreach (var diff in pair.Value.diffs)
-                                {
-                                    var index = ScoreSaberDatabaseDownloader.ScoreSaberDataFile.SongHashToScoreSaberData[pair.Key].diffs.FindIndex(x => x.diff == diff.diff);
-                                    if (index < 0)
-                                    {
-                                        ScoreSaberDatabaseDownloader.ScoreSaberDataFile.SongHashToScoreSaberData[pair.Key].diffs.Add(diff);
-                                    }
-                                }
-                            }
-                        }
-                    }
-                    Logger.Info("Success downloading ScoreSaber data!");
-                }
-                catch (System.InvalidOperationException)
-                {
-                    Logger.Error("Failed to download ScoreSaber data file...");
-                }
-                catch (Exception e)
-                {
-                    Logger.Exception("Exception trying to download ScoreSaber data file...", e);
-                }
-            }
-        }
-    }
-}

SongBrowserPlugin/manifest.json

@@ -5,8 +5,11 @@
   "gameVersion": "1.1.0",
   "id": "SongBrowser",
   "name": "Song Browser",
-  "version": "5.2.3",
+  "version": "5.3.0",
   "dependsOn": {
+    "CustomUI": "^1.5.4",
+    "SongCore": "^2.0.0",
+    "SongDataCore":  "^1.0.0"
   },
   "features": [
     "print",