From 0ebc235a933d8a514daa4f538665531ec318476e Mon Sep 17 00:00:00 2001 From: Mikhail Date: Mon, 29 Jul 2024 10:06:42 -0400 Subject: [PATCH] improve performance and ram usage --- nksrv/StaticInfo/JsonStaticData.cs | 51 +++++- nksrv/StaticInfo/StaticDataParser.cs | 233 ++++++++++++--------------- 2 files changed, 149 insertions(+), 135 deletions(-) diff --git a/nksrv/StaticInfo/JsonStaticData.cs b/nksrv/StaticInfo/JsonStaticData.cs index a1f5d4a..c001032 100644 --- a/nksrv/StaticInfo/JsonStaticData.cs +++ b/nksrv/StaticInfo/JsonStaticData.cs @@ -1,6 +1,6 @@ namespace nksrv.StaticInfo { - public class MainQuestCompletionData + public class MainQuestCompletionRecord { public int id; public int group_id; @@ -10,6 +10,10 @@ public int reward_id = 0; public int target_chapter_id; } + public class MainQuestCompletionTable + { + public List records; + } public class CampaignStageRecord { public int id; @@ -24,6 +28,10 @@ public string enter_scenario = ""; public string exit_scenario = ""; } + public class CampaignStageTable + { + public List records; + } public class RewardTableRecord { public int id; @@ -31,6 +39,11 @@ public int character_exp; public RewardEntry[]? rewards; } + public class RewardTable + { + public List records; + } + public class RewardEntry { /// @@ -53,6 +66,10 @@ public int NextId; public bool SaveTutorial; } + public class TutorialTable + { + public List records; + } public class CharacterLevelData { @@ -85,4 +102,36 @@ public int Id; public int GroupId; } + + public class CampaignChapterRecord + { + public int id; + public int chapter; + public string field_id; + public string hard_field_id; + } + public class CampaignChapterTable + { + public List records; + } + + public class CharacterRecord + { + public int id; + // TODO: There is more stuff here but it isn't needed yet + } + public class CharacterTable + { + public List records; + } + + public class ItemEquipRecord + { + public int id; + public string item_sub_type; + } + public class ItemEquipTable + { + public List records; + } } diff --git a/nksrv/StaticInfo/StaticDataParser.cs b/nksrv/StaticInfo/StaticDataParser.cs index ed8108f..f908405 100644 --- a/nksrv/StaticInfo/StaticDataParser.cs +++ b/nksrv/StaticInfo/StaticDataParser.cs @@ -3,6 +3,8 @@ using Newtonsoft.Json; using Newtonsoft.Json.Linq; using nksrv.Utils; using Swan.Logging; +using System.Collections.Generic; +using System.Diagnostics; using System.Security.Cryptography; namespace nksrv.StaticInfo @@ -37,15 +39,15 @@ namespace nksrv.StaticInfo } private ZipFile MainZip; private MemoryStream ZipStream; - private JArray questDataRecords; - private JArray stageDataRecords; - private JArray rewardDataRecords; + private Dictionary questDataRecords; + private Dictionary stageDataRecords; + private Dictionary rewardDataRecords; private JArray userExpDataRecords; - private JArray chapterCampaignData; + private Dictionary chapterCampaignData; private JArray characterCostumeTable; - private JArray characterTable; - private JArray tutorialTable; - private JArray itemEquipTable; + private Dictionary characterTable; + private Dictionary tutorialTable; + private Dictionary itemEquipTable; private Dictionary FieldMapData = []; private Dictionary LevelData = []; private Dictionary TacticAcademyLessons = []; @@ -60,8 +62,12 @@ namespace nksrv.StaticInfo await Load(); Logger.Info("Loading static data"); + var stopWatch = new Stopwatch(); + stopWatch.Start(); await Instance.Parse(); + stopWatch.Stop(); + Console.WriteLine("Parsing took " + stopWatch.Elapsed); return Instance; } @@ -211,9 +217,33 @@ namespace nksrv.StaticInfo _instance = new(targetFile); } #endregion + private async Task LoadZip(string entry, ProgressBar bar) + { + var st = new Stopwatch(); + st.Start(); + var mainQuestData = MainZip.GetEntry(entry); + if (mainQuestData == null) throw new Exception(entry + " does not exist in static data"); + using StreamReader mainQuestReader = new StreamReader(MainZip.GetInputStream(mainQuestData)); + var mainQuestDataString = await mainQuestReader.ReadToEndAsync(); + + + var questdata = JsonConvert.DeserializeObject(mainQuestDataString); + if (questdata == null) throw new Exception("failed to parse " + entry); + + currentFile++; + bar.Report((double)currentFile / totalFiles); + + st.Stop(); + Console.WriteLine($"LoadingNew {entry} took " + st.Elapsed); + + return questdata; + } private async Task LoadZip(string entry, ProgressBar bar) { + var st = new Stopwatch(); + st.Start(); + var mainQuestData = MainZip.GetEntry(entry); if (mainQuestData == null) throw new Exception(entry + " does not exist in static data"); @@ -230,6 +260,9 @@ namespace nksrv.StaticInfo currentFile++; bar.Report((double)currentFile / totalFiles); + st.Stop(); + Console.WriteLine($"LoadingOld {entry} took " + st.Elapsed); + return records; } int totalFiles = 12; @@ -238,15 +271,51 @@ namespace nksrv.StaticInfo { using var progress = new ProgressBar(); - questDataRecords = await LoadZip("MainQuestTable.json", progress); - stageDataRecords = await LoadZip("CampaignStageTable.json", progress); - rewardDataRecords = await LoadZip("RewardTable.json", progress); + var questDataRecords = await LoadZip("MainQuestTable.json", progress); + foreach (var obj in questDataRecords.records) + { + this.questDataRecords.Add(obj.id, obj); + } + + var stageDataRecords = await LoadZip("CampaignStageTable.json", progress); + foreach (var obj in stageDataRecords.records) + { + this.stageDataRecords.Add(obj.id, obj); + } + + var rewardDataRecords = await LoadZip("RewardTable.json", progress); + foreach (var obj in rewardDataRecords.records) + { + this.rewardDataRecords.Add(obj.id, obj); + } + + var chapterCampaignData = await LoadZip("CampaignChapterTable.json", progress); + foreach (var obj in chapterCampaignData.records) + { + this.chapterCampaignData.Add(obj.chapter, obj); + } + userExpDataRecords = await LoadZip("UserExpTable.json", progress); - chapterCampaignData = await LoadZip("CampaignChapterTable.json", progress); characterCostumeTable = await LoadZip("CharacterCostumeTable.json", progress); - characterTable = await LoadZip("CharacterTable.json", progress); - tutorialTable = await LoadZip("ContentsTutorialTable.json", progress); - itemEquipTable = await LoadZip("ItemEquipTable.json", progress); + + var characterTable = await LoadZip("CharacterTable.json", progress); + foreach (var obj in characterTable.records) + { + this.characterTable.Add(obj.id, obj); + } + + var tutorialTable = await LoadZip("ContentsTutorialTable.json", progress); + foreach (var obj in tutorialTable.records) + { + this.tutorialTable.Add(obj.id, obj); + } + + var itemEquipTable = await LoadZip("ItemEquipTable.json", progress); + foreach (var obj in itemEquipTable.records) + { + this.itemEquipTable.Add(obj.id, obj); + } + var characterLevelTable = await LoadZip("CharacterLevelTable.json", progress); foreach (JToken item in characterLevelTable) @@ -299,77 +368,29 @@ namespace nksrv.StaticInfo } } - public MainQuestCompletionData? GetMainQuestForStageClearCondition(int stage) + public MainQuestCompletionRecord? GetMainQuestForStageClearCondition(int stage) { - foreach (JObject item in questDataRecords) + foreach (var item in questDataRecords) { - var id = item["condition_id"]; - if (id == null) throw new Exception("expected condition_id field in quest data"); - - int value = id.ToObject(); - if (value == stage) + if (item.Value.condition_id == stage) { - MainQuestCompletionData? data = item.ToObject(); - if (data == null) throw new Exception("failed to deserialize main quest data item"); - return data; + return item.Value; } } return null; } - public MainQuestCompletionData? GetMainQuestByTableId(int tid) + public MainQuestCompletionRecord? GetMainQuestByTableId(int tid) { - foreach (JObject item in questDataRecords) - { - var id = item["id"]; - if (id == null) throw new Exception("expected condition_id field in quest data"); - - int value = id.ToObject(); - if (value == tid) - { - MainQuestCompletionData? data = item.ToObject(); - if (data == null) throw new Exception("failed to deserialize main quest data item"); - return data; - } - } - - return null; + return questDataRecords[tid]; } public CampaignStageRecord? GetStageData(int stage) { - foreach (JObject item in stageDataRecords) - { - var id = item["id"]; - if (id == null) throw new Exception("expected id field in campaign data"); - - int value = id.ToObject(); - if (value == stage) - { - CampaignStageRecord? data = JsonConvert.DeserializeObject(item.ToString()); - if (data == null) throw new Exception("failed to deserialize stage data"); - return data; - } - } - - return null; + return stageDataRecords[stage]; } public RewardTableRecord? GetRewardTableEntry(int rewardId) { - foreach (JObject item in rewardDataRecords) - { - var id = item["id"]; - if (id == null) throw new Exception("expected id field in reward data"); - - int value = id.ToObject(); - if (value == rewardId) - { - RewardTableRecord? data = JsonConvert.DeserializeObject(item.ToString()); - if (data == null) throw new Exception("failed to deserialize reward data"); - return data; - } - } - - return null; + return rewardDataRecords[rewardId]; } /// /// Returns the level and its minimum value for XP value @@ -430,24 +451,11 @@ namespace nksrv.StaticInfo } public int GetNormalChapterNumberFromFieldName(string field) { - foreach (JObject item in chapterCampaignData) + foreach (var item in chapterCampaignData) { - var id = item["field_id"]; - if (id == null) + if (item.Value.field_id == field) { - throw new Exception("expected id field in reward data"); - } - - string? value = id.ToObject(); - if (value == field) - { - var chapter = item["chapter"]; - if (chapter == null) - { - throw new Exception("expected id field in reward data"); - } - - return chapter.ToObject(); + return item.Value.chapter; } } @@ -456,14 +464,7 @@ namespace nksrv.StaticInfo public IEnumerable GetAllCharacterTids() { - foreach (JObject item in characterTable) - { - var id = item["id"]; - if (id == null) throw new Exception("expected id field in reward data"); - - int value = id.ToObject(); - yield return value; - } + return characterTable.Keys; } public IEnumerable GetAllCostumes() { @@ -479,56 +480,20 @@ namespace nksrv.StaticInfo internal ClearedTutorialData GetTutorialDataById(int TableId) { - foreach (JObject item in tutorialTable) - { - var id = item["id"]; - if (id == null) - { - throw new Exception("expected id field in reward data"); - } - - int idValue = id.ToObject(); - if (idValue == TableId) - { - ClearedTutorialData? data = item.ToObject(); - if (data == null) throw new Exception("failed to deserialize reward data"); - return data; - } - } - - throw new Exception("tutorial not found: " + TableId); + return tutorialTable[TableId]; } public string? GetItemSubType(int itemType) { - foreach (JObject item in itemEquipTable) - { - var id = item["id"]; - if (id == null) throw new Exception("expected id field in reward data"); - - int? idValue = id.ToObject(); - if (idValue == itemType) - { - var subtype = item["item_sub_type"]; - if (subtype == null) - { - throw new Exception("expected item_sub_type field in item equip data"); - } - - return subtype.ToObject(); - } - } - - return null; + return itemEquipTable[itemType].item_sub_type; } internal IEnumerable GetStageIdsForChapter(int chapterNumber, bool normal) { string mod = normal ? "Normal" : "Hard"; - foreach (JObject item in stageDataRecords) + foreach (var item in stageDataRecords) { - CampaignStageRecord? data = item.ToObject(); - if (data == null) throw new Exception("failed to deserialize stage data"); + var data = item.Value; int chVal = data.chapter_id - 1;