improve performance and ram usage

nksrv/StaticInfo/JsonStaticData.cs

@@ -1,6 +1,6 @@
 namespace nksrv.StaticInfo
 {
-    public class MainQuestCompletionData
+    public class MainQuestCompletionRecord
     {
         public int id;
         public int group_id;
@@ -10,6 +10,10 @@
         public int reward_id = 0;
         public int target_chapter_id;
     }
+    public class MainQuestCompletionTable
+    {
+        public List<MainQuestCompletionRecord> records;
+    }
     public class CampaignStageRecord
     {
         public int id;
@@ -24,6 +28,10 @@
         public string enter_scenario = "";
         public string exit_scenario = "";
     }
+    public class CampaignStageTable
+    {
+        public List<CampaignStageRecord> records;
+    }
     public class RewardTableRecord
     {
         public int id;
@@ -31,6 +39,11 @@
         public int character_exp;
         public RewardEntry[]? rewards;
     }
+    public class RewardTable
+    {
+        public List<RewardTableRecord> records;
+    }
+
     public class RewardEntry
     {
         /// <summary>
@@ -53,6 +66,10 @@
         public int NextId;
         public bool SaveTutorial;
     }
+    public class TutorialTable
+    {
+        public List<ClearedTutorialData> records;
+    }
 
     public class CharacterLevelData
     {
@@ -85,4 +102,36 @@
         public int Id;
         public int GroupId;
     }
+
+    public class CampaignChapterRecord
+    {
+        public int id;
+        public int chapter;
+        public string field_id;
+        public string hard_field_id;
+    }
+    public class CampaignChapterTable
+    {
+        public List<CampaignChapterRecord> records;
+    }
+
+    public class CharacterRecord
+    {
+        public int id;
+        // TODO: There is more stuff here but it isn't needed yet
+    }
+    public class CharacterTable
+    {
+        public List<CharacterRecord> records;
+    }
+
+    public class ItemEquipRecord
+    {
+        public int id;
+        public string item_sub_type;
+    }
+    public class ItemEquipTable
+    {
+        public List<ItemEquipRecord> records;
+    }
 }

nksrv/StaticInfo/StaticDataParser.cs

@@ -3,6 +3,8 @@ using Newtonsoft.Json;
 using Newtonsoft.Json.Linq;
 using nksrv.Utils;
 using Swan.Logging;
+using System.Collections.Generic;
+using System.Diagnostics;
 using System.Security.Cryptography;
 
 namespace nksrv.StaticInfo
@@ -37,15 +39,15 @@ namespace nksrv.StaticInfo
         }
         private ZipFile MainZip;
         private MemoryStream ZipStream;
-        private JArray questDataRecords;
+        private Dictionary<int, MainQuestCompletionRecord> questDataRecords;
-        private JArray stageDataRecords;
+        private Dictionary<int, CampaignStageRecord> stageDataRecords;
-        private JArray rewardDataRecords;
+        private Dictionary<int, RewardTableRecord> rewardDataRecords;
         private JArray userExpDataRecords;
-        private JArray chapterCampaignData;
+        private Dictionary<int, CampaignChapterRecord> chapterCampaignData;
         private JArray characterCostumeTable;
-        private JArray characterTable;
+        private Dictionary<int, CharacterRecord> characterTable;
-        private JArray tutorialTable;
+        private Dictionary<int, ClearedTutorialData> tutorialTable;
-        private JArray itemEquipTable;
+        private Dictionary<int, ItemEquipRecord> itemEquipTable;
         private Dictionary<string, JArray> FieldMapData = [];
         private Dictionary<int, CharacterLevelData> LevelData = [];
         private Dictionary<int, TacticAcademyLessonRecord> TacticAcademyLessons = [];
@@ -60,8 +62,12 @@ namespace nksrv.StaticInfo
             await Load();
 
             Logger.Info("Loading static data");
+            var stopWatch = new Stopwatch();
+            stopWatch.Start();
             await Instance.Parse();
 
+            stopWatch.Stop();
+            Console.WriteLine("Parsing took " + stopWatch.Elapsed);
             return Instance;
         }
 
@@ -211,9 +217,33 @@ namespace nksrv.StaticInfo
             _instance = new(targetFile);
         }
         #endregion
+        private async Task<T> LoadZip<T>(string entry, ProgressBar bar)
+        {
+            var st = new Stopwatch();
+            st.Start();
+            var mainQuestData = MainZip.GetEntry(entry);
+            if (mainQuestData == null) throw new Exception(entry + " does not exist in static data");
 
+            using StreamReader mainQuestReader = new StreamReader(MainZip.GetInputStream(mainQuestData));
+            var mainQuestDataString = await mainQuestReader.ReadToEndAsync();
+
+
+            var questdata = JsonConvert.DeserializeObject<T>(mainQuestDataString);
+            if (questdata == null) throw new Exception("failed to parse " + entry);
+
+            currentFile++;
+            bar.Report((double)currentFile / totalFiles);
+
+            st.Stop();
+            Console.WriteLine($"LoadingNew {entry} took " + st.Elapsed);
+
+            return questdata;
+        }
         private async Task<JArray> LoadZip(string entry, ProgressBar bar)
         {
+            var st = new Stopwatch();
+            st.Start();
+
             var mainQuestData = MainZip.GetEntry(entry);
             if (mainQuestData == null) throw new Exception(entry + " does not exist in static data");
 
@@ -230,6 +260,9 @@ namespace nksrv.StaticInfo
             currentFile++;
             bar.Report((double)currentFile / totalFiles);
 
+            st.Stop();
+            Console.WriteLine($"LoadingOld {entry} took " + st.Elapsed);
+
             return records;
         }
         int totalFiles = 12;
@@ -238,15 +271,51 @@ namespace nksrv.StaticInfo
         {
             using var progress = new ProgressBar();
 
-            questDataRecords = await LoadZip("MainQuestTable.json", progress);
+            var questDataRecords = await LoadZip<MainQuestCompletionTable>("MainQuestTable.json", progress);
-            stageDataRecords = await LoadZip("CampaignStageTable.json", progress);
+            foreach (var obj in questDataRecords.records)
-            rewardDataRecords = await LoadZip("RewardTable.json", progress);
+            {
+                this.questDataRecords.Add(obj.id, obj);
+            }
+
+            var stageDataRecords = await LoadZip<CampaignStageTable>("CampaignStageTable.json", progress);
+            foreach (var obj in stageDataRecords.records)
+            {
+                this.stageDataRecords.Add(obj.id, obj);
+            }
+
+            var rewardDataRecords = await LoadZip<RewardTable>("RewardTable.json", progress);
+            foreach (var obj in rewardDataRecords.records)
+            {
+                this.rewardDataRecords.Add(obj.id, obj);
+            }
+
+            var chapterCampaignData = await LoadZip<CampaignChapterTable>("CampaignChapterTable.json", progress);
+            foreach (var obj in chapterCampaignData.records)
+            {
+                this.chapterCampaignData.Add(obj.chapter, obj);
+            }
+
             userExpDataRecords = await LoadZip("UserExpTable.json", progress);
-            chapterCampaignData = await LoadZip("CampaignChapterTable.json", progress);
             characterCostumeTable = await LoadZip("CharacterCostumeTable.json", progress);
-            characterTable = await LoadZip("CharacterTable.json", progress);
+
-            tutorialTable = await LoadZip("ContentsTutorialTable.json", progress);
+            var characterTable = await LoadZip<CharacterTable>("CharacterTable.json", progress);
-            itemEquipTable = await LoadZip("ItemEquipTable.json", progress);
+            foreach (var obj in characterTable.records)
+            {
+                this.characterTable.Add(obj.id, obj);
+            }
+
+            var tutorialTable = await LoadZip<TutorialTable>("ContentsTutorialTable.json", progress);
+            foreach (var obj in tutorialTable.records)
+            {
+                this.tutorialTable.Add(obj.id, obj);
+            }
+
+            var itemEquipTable = await LoadZip<ItemEquipTable>("ItemEquipTable.json", progress);
+            foreach (var obj in itemEquipTable.records)
+            {
+                this.itemEquipTable.Add(obj.id, obj);
+            }
+
             var characterLevelTable = await LoadZip("CharacterLevelTable.json", progress);
 
             foreach (JToken item in characterLevelTable)
@@ -299,77 +368,29 @@ namespace nksrv.StaticInfo
             }
         }
 
-        public MainQuestCompletionData? GetMainQuestForStageClearCondition(int stage)
+        public MainQuestCompletionRecord? GetMainQuestForStageClearCondition(int stage)
         {
-            foreach (JObject item in questDataRecords)
+            foreach (var item in questDataRecords)
             {
-                var id = item["condition_id"];
+                if (item.Value.condition_id == stage)
-                if (id == null) throw new Exception("expected condition_id field in quest data");
-
-                int value = id.ToObject<int>();
-                if (value == stage)
                 {
-                    MainQuestCompletionData? data = item.ToObject<MainQuestCompletionData>();
+                    return item.Value;
-                    if (data == null) throw new Exception("failed to deserialize main quest data item");
-                    return data;
                 }
             }
 
             return null;
         }
-        public MainQuestCompletionData? GetMainQuestByTableId(int tid)
+        public MainQuestCompletionRecord? GetMainQuestByTableId(int tid)
         {
-            foreach (JObject item in questDataRecords)
+            return questDataRecords[tid];
-            {
-                var id = item["id"];
-                if (id == null) throw new Exception("expected condition_id field in quest data");
-
-                int value = id.ToObject<int>();
-                if (value == tid)
-                {
-                    MainQuestCompletionData? data = item.ToObject<MainQuestCompletionData>();
-                    if (data == null) throw new Exception("failed to deserialize main quest data item");
-                    return data;
-                }
-            }
-
-            return null;
         }
         public CampaignStageRecord? GetStageData(int stage)
         {
-            foreach (JObject item in stageDataRecords)
+            return stageDataRecords[stage];
-            {
-                var id = item["id"];
-                if (id == null) throw new Exception("expected id field in campaign data");
-
-                int value = id.ToObject<int>();
-                if (value == stage)
-                {
-                    CampaignStageRecord? data = JsonConvert.DeserializeObject<CampaignStageRecord>(item.ToString());
-                    if (data == null) throw new Exception("failed to deserialize stage data");
-                    return data;
-                }
-            }
-
-            return null;
         }
         public RewardTableRecord? GetRewardTableEntry(int rewardId)
         {
-            foreach (JObject item in rewardDataRecords)
+            return rewardDataRecords[rewardId];
-            {
-                var id = item["id"];
-                if (id == null) throw new Exception("expected id field in reward data");
-
-                int value = id.ToObject<int>();
-                if (value == rewardId)
-                {
-                    RewardTableRecord? data = JsonConvert.DeserializeObject<RewardTableRecord>(item.ToString());
-                    if (data == null) throw new Exception("failed to deserialize reward data");
-                    return data;
-                }
-            }
-
-            return null;
         }
         /// <summary>
         /// Returns the level and its minimum value for XP value
@@ -430,24 +451,11 @@ namespace nksrv.StaticInfo
         }
         public int GetNormalChapterNumberFromFieldName(string field)
         {
-            foreach (JObject item in chapterCampaignData)
+            foreach (var item in chapterCampaignData)
             {
-                var id = item["field_id"];
+                if (item.Value.field_id == field)
-                if (id == null)
                 {
-                    throw new Exception("expected id field in reward data");
+                    return item.Value.chapter;
-                }
-
-                string? value = id.ToObject<string>();
-                if (value == field)
-                {
-                    var chapter = item["chapter"];
-                    if (chapter == null)
-                    {
-                        throw new Exception("expected id field in reward data");
-                    }
-
-                    return chapter.ToObject<int>();
                 }
             }
 
@@ -456,14 +464,7 @@ namespace nksrv.StaticInfo
 
         public IEnumerable<int> GetAllCharacterTids()
         {
-            foreach (JObject item in characterTable)
+            return characterTable.Keys;
-            {
-                var id = item["id"];
-                if (id == null) throw new Exception("expected id field in reward data");
-
-                int value = id.ToObject<int>();
-                yield return value;
-            }
         }
         public IEnumerable<int> GetAllCostumes()
         {
@@ -479,56 +480,20 @@ namespace nksrv.StaticInfo
 
         internal ClearedTutorialData GetTutorialDataById(int TableId)
         {
-            foreach (JObject item in tutorialTable)
+            return tutorialTable[TableId];
-            {
-                var id = item["id"];
-                if (id == null)
-                {
-                    throw new Exception("expected id field in reward data");
-                }
-
-                int idValue = id.ToObject<int>();
-                if (idValue == TableId)
-                {
-                    ClearedTutorialData? data = item.ToObject<ClearedTutorialData>();
-                    if (data == null) throw new Exception("failed to deserialize reward data");
-                    return data;
-                }
-            }
-
-            throw new Exception("tutorial not found: " + TableId);
         }
 
         public string? GetItemSubType(int itemType)
         {
-            foreach (JObject item in itemEquipTable)
+            return itemEquipTable[itemType].item_sub_type;
-            {
-                var id = item["id"];
-                if (id == null) throw new Exception("expected id field in reward data");
-
-                int? idValue = id.ToObject<int>();
-                if (idValue == itemType)
-                {
-                    var subtype = item["item_sub_type"];
-                    if (subtype == null)
-                    {
-                        throw new Exception("expected item_sub_type field in item equip data");
-                    }
-
-                    return subtype.ToObject<string>();
-                }
-            }
-
-            return null;
         }
 
         internal IEnumerable<int> GetStageIdsForChapter(int chapterNumber, bool normal)
         {
             string mod = normal ? "Normal" : "Hard";
-            foreach (JObject item in stageDataRecords)
+            foreach (var item in stageDataRecords)
             {
-                CampaignStageRecord? data = item.ToObject<CampaignStageRecord>();
+                var data = item.Value;
-                if (data == null) throw new Exception("failed to deserialize stage data");
 
                 int chVal = data.chapter_id - 1;