From 36a35c11aab583360e9035f716d3fb23450e0b20 Mon Sep 17 00:00:00 2001 From: KingRainbow44 Date: Tue, 30 May 2023 15:38:26 -0400 Subject: [PATCH] Use thread executors to speed up the database loading process --- .../java/emu/grasscutter/Grasscutter.java | 31 ++- .../grasscutter/database/DatabaseHelper.java | 190 ++++++++++++------ .../grasscutter/database/DatabaseManager.java | 7 +- .../grasscutter/game/inventory/Inventory.java | 8 +- .../emu/grasscutter/game/player/Player.java | 19 +- .../grasscutter/utils/objects/Returnable.java | 8 + 6 files changed, 184 insertions(+), 79 deletions(-) create mode 100644 src/main/java/emu/grasscutter/utils/objects/Returnable.java diff --git a/src/main/java/emu/grasscutter/Grasscutter.java b/src/main/java/emu/grasscutter/Grasscutter.java index 0846dd39b..04bcae141 100644 --- a/src/main/java/emu/grasscutter/Grasscutter.java +++ b/src/main/java/emu/grasscutter/Grasscutter.java @@ -1,8 +1,5 @@ package emu.grasscutter; -import static emu.grasscutter.config.Configuration.SERVER; -import static emu.grasscutter.utils.lang.Language.translate; - import ch.qos.logback.classic.Level; import ch.qos.logback.classic.Logger; import emu.grasscutter.auth.AuthenticationSystem; @@ -33,12 +30,7 @@ import emu.grasscutter.utils.JsonUtils; import emu.grasscutter.utils.StartupArguments; import emu.grasscutter.utils.Utils; import emu.grasscutter.utils.lang.Language; -import java.io.File; -import java.io.FileWriter; -import java.io.IOError; -import java.io.IOException; -import java.util.Calendar; -import javax.annotation.Nullable; +import io.netty.util.concurrent.FastThreadLocalThread; import lombok.Getter; import lombok.Setter; import org.jline.reader.EndOfFileException; @@ -50,6 +42,20 @@ import org.jline.terminal.TerminalBuilder; import org.reflections.Reflections; import org.slf4j.LoggerFactory; +import javax.annotation.Nullable; +import java.io.File; +import java.io.FileWriter; +import java.io.IOError; +import java.io.IOException; +import java.util.Calendar; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.LinkedBlockingDeque; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +import static emu.grasscutter.config.Configuration.SERVER; +import static emu.grasscutter.utils.lang.Language.translate; + public final class Grasscutter { public static final File configFile = new File("./config.json"); public static final Reflections reflector = new Reflections("emu.grasscutter"); @@ -75,6 +81,13 @@ public final class Grasscutter { private static LineReader consoleLineReader = null; + @Getter private static final ExecutorService threadPool = new ThreadPoolExecutor( + 6, 6, 60, TimeUnit.SECONDS, + new LinkedBlockingDeque<>(), + FastThreadLocalThread::new, + new ThreadPoolExecutor.AbortPolicy() + ); + static { // Declare logback configuration. System.setProperty("logback.configurationFile", "src/main/resources/logback.xml"); diff --git a/src/main/java/emu/grasscutter/database/DatabaseHelper.java b/src/main/java/emu/grasscutter/database/DatabaseHelper.java index dfac8da12..9fd515aac 100644 --- a/src/main/java/emu/grasscutter/database/DatabaseHelper.java +++ b/src/main/java/emu/grasscutter/database/DatabaseHelper.java @@ -1,8 +1,5 @@ package emu.grasscutter.database; -import static com.mongodb.client.model.Filters.eq; - -import com.mongodb.client.result.DeleteResult; import dev.morphia.query.FindOptions; import dev.morphia.query.Sort; import dev.morphia.query.experimental.filters.Filters; @@ -22,10 +19,74 @@ import emu.grasscutter.game.mail.Mail; import emu.grasscutter.game.player.Player; import emu.grasscutter.game.quest.GameMainQuest; import emu.grasscutter.game.world.SceneGroupInstance; +import emu.grasscutter.utils.objects.Returnable; +import io.netty.util.concurrent.FastThreadLocalThread; + import java.util.List; +import java.util.concurrent.*; import java.util.stream.Stream; +import static com.mongodb.client.model.Filters.eq; + public final class DatabaseHelper { + private static final ExecutorService eventExecutor = new ThreadPoolExecutor( + 6, 6, 60, TimeUnit.SECONDS, + new LinkedBlockingDeque<>(), + FastThreadLocalThread::new, + new ThreadPoolExecutor.AbortPolicy() + ); + + /** + * Saves an object on the account datastore. + * + * @param object The object to save. + */ + public static void saveAccountAsync(Object object) { + DatabaseHelper.eventExecutor.submit(() -> + DatabaseManager.getAccountDatastore().save(object)); + } + + /** + * Saves an object on the game datastore. + * + * @param object The object to save. + */ + public static void saveGameAsync(Object object) { + DatabaseHelper.eventExecutor.submit(() -> + DatabaseHelper.saveGameAsync(object)); + } + + /** + * Runs a runnable on the event executor. + * Should be limited to database-related operations. + * + * @param runnable The runnable to run. + */ + public static void asyncOperation(Runnable runnable) { + DatabaseHelper.eventExecutor.submit(runnable); + } + + /** + * Fetches an object asynchronously. + * + * @param task The task to run. + * @return The future. + */ + public static CompletableFuture fetchAsync(Returnable task) { + var future = new CompletableFuture(); + + // Run the task on the event executor. + DatabaseHelper.eventExecutor.submit(() -> { + try { + future.complete(task.invoke()); + } catch (Exception e) { + future.completeExceptionally(e); + } + }); + + return future; + } + public static Account createAccount(String username) { return createAccountWithUid(username, 0); } @@ -85,7 +146,7 @@ public final class DatabaseHelper { } public static void saveAccount(Account account) { - DatabaseManager.getAccountDatastore().save(account); + DatabaseHelper.saveAccountAsync(account); } public static Account getAccountByName(String username) { @@ -159,31 +220,34 @@ public final class DatabaseHelper { if (player == null) return; } int uid = player.getUid(); - // Delete data from collections - DatabaseManager.getGameDatabase().getCollection("achievements").deleteMany(eq("uid", uid)); - DatabaseManager.getGameDatabase().getCollection("activities").deleteMany(eq("uid", uid)); - DatabaseManager.getGameDatabase().getCollection("homes").deleteMany(eq("ownerUid", uid)); - DatabaseManager.getGameDatabase().getCollection("mail").deleteMany(eq("ownerUid", uid)); - DatabaseManager.getGameDatabase().getCollection("avatars").deleteMany(eq("ownerId", uid)); - DatabaseManager.getGameDatabase().getCollection("gachas").deleteMany(eq("ownerId", uid)); - DatabaseManager.getGameDatabase().getCollection("items").deleteMany(eq("ownerId", uid)); - DatabaseManager.getGameDatabase().getCollection("quests").deleteMany(eq("ownerUid", uid)); - DatabaseManager.getGameDatabase().getCollection("battlepass").deleteMany(eq("ownerUid", uid)); - // Delete friendships. - // Here, we need to make sure to not only delete the deleted account's friendships, - // but also all friendship entries for that account's friends. - DatabaseManager.getGameDatabase().getCollection("friendships").deleteMany(eq("ownerId", uid)); - DatabaseManager.getGameDatabase().getCollection("friendships").deleteMany(eq("friendId", uid)); + DatabaseHelper.asyncOperation(() -> { + // Delete data from collections + DatabaseManager.getGameDatabase().getCollection("achievements").deleteMany(eq("uid", uid)); + DatabaseManager.getGameDatabase().getCollection("activities").deleteMany(eq("uid", uid)); + DatabaseManager.getGameDatabase().getCollection("homes").deleteMany(eq("ownerUid", uid)); + DatabaseManager.getGameDatabase().getCollection("mail").deleteMany(eq("ownerUid", uid)); + DatabaseManager.getGameDatabase().getCollection("avatars").deleteMany(eq("ownerId", uid)); + DatabaseManager.getGameDatabase().getCollection("gachas").deleteMany(eq("ownerId", uid)); + DatabaseManager.getGameDatabase().getCollection("items").deleteMany(eq("ownerId", uid)); + DatabaseManager.getGameDatabase().getCollection("quests").deleteMany(eq("ownerUid", uid)); + DatabaseManager.getGameDatabase().getCollection("battlepass").deleteMany(eq("ownerUid", uid)); - // Delete the player last. - DatabaseManager.getGameDatastore().find(Player.class).filter(Filters.eq("id", uid)).delete(); + // Delete friendships. + // Here, we need to make sure to not only delete the deleted account's friendships, + // but also all friendship entries for that account's friends. + DatabaseManager.getGameDatabase().getCollection("friendships").deleteMany(eq("ownerId", uid)); + DatabaseManager.getGameDatabase().getCollection("friendships").deleteMany(eq("friendId", uid)); - // Finally, delete the account itself. - DatabaseManager.getAccountDatastore() + // Delete the player last. + DatabaseManager.getGameDatastore().find(Player.class).filter(Filters.eq("id", uid)).delete(); + + // Finally, delete the account itself. + DatabaseManager.getAccountDatastore() .find(Account.class) .filter(Filters.eq("id", target.getId())) .delete(); + }); } public static Stream getByGameClass(Class classType) { @@ -239,7 +303,7 @@ public final class DatabaseHelper { > 0; } - public static synchronized Player generatePlayerUid(Player character, int reservedId) { + public static synchronized void generatePlayerUid(Player character, int reservedId) { // Check if reserved id int id; if (reservedId > 0 && !checkIfPlayerExists(reservedId)) { @@ -251,9 +315,9 @@ public final class DatabaseHelper { } while (checkIfPlayerExists(id)); character.setUid(id); } + // Save to database - DatabaseManager.getGameDatastore().save(character); - return character; + DatabaseHelper.saveGameAsync(character); } public static synchronized int getNextPlayerId(int reservedId) { @@ -270,36 +334,48 @@ public final class DatabaseHelper { } public static void savePlayer(Player character) { - DatabaseManager.getGameDatastore().save(character); + DatabaseHelper.saveGameAsync(character); } public static void saveAvatar(Avatar avatar) { - DatabaseManager.getGameDatastore().save(avatar); + DatabaseHelper.saveGameAsync(avatar); } + /** + * Fetches all avatars of a player. + * + * @param player The player. + * @return The list of avatars. + */ public static List getAvatars(Player player) { return DatabaseManager.getGameDatastore() - .find(Avatar.class) - .filter(Filters.eq("ownerId", player.getUid())) - .stream() - .toList(); + .find(Avatar.class) + .filter(Filters.eq("ownerId", player.getUid())) + .stream() + .toList(); } public static void saveItem(GameItem item) { - DatabaseManager.getGameDatastore().save(item); + DatabaseHelper.saveGameAsync(item); } - public static boolean deleteItem(GameItem item) { - DeleteResult result = DatabaseManager.getGameDatastore().delete(item); - return result.wasAcknowledged(); + public static void deleteItem(GameItem item) { + DatabaseHelper.asyncOperation(() -> + DatabaseManager.getGameDatastore().delete(item)); } + /** + * Fetches all items of a player. + * + * @param player The player. + * @return The list of items. + */ public static List getInventoryItems(Player player) { return DatabaseManager.getGameDatastore() - .find(GameItem.class) - .filter(Filters.eq("ownerId", player.getUid())) - .stream() - .toList(); + .find(GameItem.class) + .filter(Filters.eq("ownerId", player.getUid())) + .stream() + .toList(); } public static List getFriends(Player player) { @@ -319,11 +395,12 @@ public final class DatabaseHelper { } public static void saveFriendship(Friendship friendship) { - DatabaseManager.getGameDatastore().save(friendship); + DatabaseHelper.saveGameAsync(friendship); } public static void deleteFriendship(Friendship friendship) { - DatabaseManager.getGameDatastore().delete(friendship); + DatabaseHelper.asyncOperation(() -> + DatabaseManager.getGameDatastore().delete(friendship)); } public static Friendship getReverseFriendship(Friendship friendship) { @@ -367,7 +444,7 @@ public final class DatabaseHelper { } public static void saveGachaRecord(GachaRecord gachaRecord) { - DatabaseManager.getGameDatastore().save(gachaRecord); + DatabaseHelper.saveGameAsync(gachaRecord); } public static List getAllMail(Player player) { @@ -379,12 +456,12 @@ public final class DatabaseHelper { } public static void saveMail(Mail mail) { - DatabaseManager.getGameDatastore().save(mail); + DatabaseHelper.saveGameAsync(mail); } - public static boolean deleteMail(Mail mail) { - DeleteResult result = DatabaseManager.getGameDatastore().delete(mail); - return result.wasAcknowledged(); + public static void deleteMail(Mail mail) { + DatabaseHelper.asyncOperation(() -> + DatabaseManager.getGameDatastore().delete(mail)); } public static List getAllQuests(Player player) { @@ -396,11 +473,12 @@ public final class DatabaseHelper { } public static void saveQuest(GameMainQuest quest) { - DatabaseManager.getGameDatastore().save(quest); + DatabaseHelper.saveGameAsync(quest); } - public static boolean deleteQuest(GameMainQuest quest) { - return DatabaseManager.getGameDatastore().delete(quest).wasAcknowledged(); + public static void deleteQuest(GameMainQuest quest) { + DatabaseHelper.asyncOperation(() -> + DatabaseManager.getGameDatastore().delete(quest)); } public static GameHome getHomeByUid(int id) { @@ -411,7 +489,7 @@ public final class DatabaseHelper { } public static void saveHome(GameHome gameHome) { - DatabaseManager.getGameDatastore().save(gameHome); + DatabaseHelper.saveGameAsync(gameHome); } public static BattlePassManager loadBattlePass(Player player) { @@ -430,7 +508,7 @@ public final class DatabaseHelper { } public static void saveBattlePass(BattlePassManager manager) { - DatabaseManager.getGameDatastore().save(manager); + DatabaseHelper.saveGameAsync(manager); } public static PlayerActivityData getPlayerActivityData(int uid, int activityId) { @@ -441,7 +519,7 @@ public final class DatabaseHelper { } public static void savePlayerActivityData(PlayerActivityData playerActivityData) { - DatabaseManager.getGameDatastore().save(playerActivityData); + DatabaseHelper.saveGameAsync(playerActivityData); } public static MusicGameBeatmap getMusicGameBeatmap(long musicShareId) { @@ -452,7 +530,7 @@ public final class DatabaseHelper { } public static void saveMusicGameBeatmap(MusicGameBeatmap musicGameBeatmap) { - DatabaseManager.getGameDatastore().save(musicGameBeatmap); + DatabaseHelper.saveGameAsync(musicGameBeatmap); } public static Achievements getAchievementData(int uid) { @@ -463,11 +541,11 @@ public final class DatabaseHelper { } public static void saveAchievementData(Achievements achievements) { - DatabaseManager.getGameDatastore().save(achievements); + DatabaseHelper.saveGameAsync(achievements); } public static void saveGroupInstance(SceneGroupInstance instance) { - DatabaseManager.getGameDatastore().save(instance); + DatabaseHelper.saveGameAsync(instance); } public static SceneGroupInstance loadGroupInstance(int groupId, Player owner) { diff --git a/src/main/java/emu/grasscutter/database/DatabaseManager.java b/src/main/java/emu/grasscutter/database/DatabaseManager.java index 45ad073fa..3f48d9135 100644 --- a/src/main/java/emu/grasscutter/database/DatabaseManager.java +++ b/src/main/java/emu/grasscutter/database/DatabaseManager.java @@ -1,7 +1,5 @@ package emu.grasscutter.database; -import static emu.grasscutter.config.Configuration.DATABASE; - import com.mongodb.MongoCommandException; import com.mongodb.client.MongoClient; import com.mongodb.client.MongoClients; @@ -18,6 +16,8 @@ import emu.grasscutter.Grasscutter.ServerRunMode; import emu.grasscutter.game.Account; import org.reflections.Reflections; +import static emu.grasscutter.config.Configuration.DATABASE; + public final class DatabaseManager { private static Datastore gameDatastore; private static Datastore dispatchDatastore; @@ -107,10 +107,11 @@ public final class DatabaseManager { if (counter == null) { counter = new DatabaseCounter(c.getSimpleName()); } + try { return counter.getNextId(); } finally { - getGameDatastore().save(counter); + DatabaseHelper.saveGameAsync(counter); } } diff --git a/src/main/java/emu/grasscutter/game/inventory/Inventory.java b/src/main/java/emu/grasscutter/game/inventory/Inventory.java index 12f5cbfba..9e791e2d6 100644 --- a/src/main/java/emu/grasscutter/game/inventory/Inventory.java +++ b/src/main/java/emu/grasscutter/game/inventory/Inventory.java @@ -1,7 +1,5 @@ package emu.grasscutter.game.inventory; -import static emu.grasscutter.config.Configuration.INVENTORY_LIMITS; - import emu.grasscutter.Grasscutter; import emu.grasscutter.data.GameData; import emu.grasscutter.data.common.ItemParamData; @@ -22,11 +20,14 @@ import it.unimi.dsi.fastutil.ints.Int2ObjectMap; import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap; import it.unimi.dsi.fastutil.longs.Long2ObjectMap; import it.unimi.dsi.fastutil.longs.Long2ObjectOpenHashMap; + import java.util.ArrayList; import java.util.Collection; import java.util.Iterator; import java.util.List; +import static emu.grasscutter.config.Configuration.INVENTORY_LIMITS; + public class Inventory extends BasePlayerManager implements Iterable { private final Long2ObjectMap store; private final Int2ObjectMap inventoryTypes; @@ -572,6 +573,9 @@ public class Inventory extends BasePlayerManager implements Iterable { } } } + + // Load avatars after inventory. + this.getPlayer().getAvatars().postLoad(); } @Override diff --git a/src/main/java/emu/grasscutter/game/player/Player.java b/src/main/java/emu/grasscutter/game/player/Player.java index 77329b223..feeacc022 100644 --- a/src/main/java/emu/grasscutter/game/player/Player.java +++ b/src/main/java/emu/grasscutter/game/player/Player.java @@ -1313,16 +1313,17 @@ public class Player implements PlayerHook, FieldFetch { } // Load from db - this.achievements = Achievements.getByPlayer(this); - this.getAvatars().loadFromDatabase(); - this.getInventory().loadFromDatabase(); + var runner = Grasscutter.getThreadPool(); + runner.submit(() -> this.achievements = Achievements.getByPlayer(this)); - this.getFriendsList().loadFromDatabase(); - this.getMailHandler().loadFromDatabase(); - this.getQuestManager().loadFromDatabase(); + runner.submit(this.getAvatars()::loadFromDatabase); + runner.submit(this.getInventory()::loadFromDatabase); - this.loadBattlePassManager(); - this.getAvatars().postLoad(); // Needs to be called after inventory is handled + runner.submit(this.getFriendsList()::loadFromDatabase); + runner.submit(this.getMailHandler()::loadFromDatabase); + runner.submit(this.getQuestManager()::loadFromDatabase); + + runner.submit(this::loadBattlePassManager); this.getPlayerProgress().setPlayer(this); // Add reference to the player. } @@ -1397,7 +1398,7 @@ public class Player implements PlayerHook, FieldFetch { home = GameHome.getByUid(getUid()); home.onOwnerLogin(this); // Activity - activityManager = new ActivityManager(this); + this.activityManager = new ActivityManager(this); session.send(new PacketPlayerEnterSceneNotify(this)); // Enter game world session.send(new PacketPlayerLevelRewardUpdateNotify(rewardedLevels)); diff --git a/src/main/java/emu/grasscutter/utils/objects/Returnable.java b/src/main/java/emu/grasscutter/utils/objects/Returnable.java new file mode 100644 index 000000000..e04127bc9 --- /dev/null +++ b/src/main/java/emu/grasscutter/utils/objects/Returnable.java @@ -0,0 +1,8 @@ +package emu.grasscutter.utils.objects; + +public interface Returnable { + /** + * @return The value. + */ + T invoke(); +}