mirror of
https://github.com/Grasscutters/Grasscutter.git
synced 2025-12-19 18:34:49 +01:00
Cache used strings from TextMaps
This commit is contained in:
AnimeGitB
@@ -3,14 +3,43 @@ package emu.grasscutter.utils;
|
||||
import com.google.gson.JsonElement;
|
||||
import com.google.gson.JsonObject;
|
||||
import emu.grasscutter.Grasscutter;
|
||||
import emu.grasscutter.data.GameData;
|
||||
import emu.grasscutter.data.ResourceLoader;
|
||||
import emu.grasscutter.game.player.Player;
|
||||
import it.unimi.dsi.fastutil.ints.Int2ObjectMap;
|
||||
import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap;
|
||||
import it.unimi.dsi.fastutil.ints.IntOpenHashSet;
|
||||
import it.unimi.dsi.fastutil.ints.IntSet;
|
||||
import it.unimi.dsi.fastutil.objects.Object2IntMap;
|
||||
import it.unimi.dsi.fastutil.objects.Object2IntOpenHashMap;
|
||||
import lombok.EqualsAndHashCode;
|
||||
|
||||
import javax.annotation.Nullable;
|
||||
|
||||
import static emu.grasscutter.config.Configuration.*;
|
||||
|
||||
import java.io.BufferedInputStream;
|
||||
import java.io.BufferedOutputStream;
|
||||
import java.io.BufferedReader;
|
||||
import java.io.FileReader;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.ObjectInputStream;
|
||||
import java.io.ObjectOutputStream;
|
||||
import java.io.Serializable;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.FileAlreadyExistsException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.StandardOpenOption;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.IntStream;
|
||||
import java.util.Arrays;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
public final class Language {
|
||||
@@ -207,4 +236,187 @@ public final class Language {
|
||||
return languageFile;
|
||||
}
|
||||
}
|
||||
|
||||
private static final int TEXTMAP_CACHE_VERSION = 0x9CCACE02;
|
||||
@EqualsAndHashCode public static class TextStrings implements Serializable {
|
||||
public static final String[] ARR_LANGUAGES = {"EN", "CHS", "CHT", "JP", "KR", "DE", "ES", "FR", "ID", "PT", "RU", "TH", "VI"};
|
||||
public static final String[] ARR_GC_LANGUAGES = {"en-US", "zh-CN", "zh-TW", "JP", "KR", "DE", "es-ES", "fr-FR", "ID", "PT", "ru-RU", "TH", "VI"};
|
||||
public static final int NUM_LANGUAGES = ARR_LANGUAGES.length;
|
||||
public static final List<String> LIST_LANGUAGES = Arrays.asList(ARR_LANGUAGES);
|
||||
public static final Object2IntMap<String> MAP_LANGUAGES = // Map "EN": 0, "CHS": 1, ..., "VI": 12
|
||||
new Object2IntOpenHashMap<>(
|
||||
IntStream.range(0, ARR_LANGUAGES.length)
|
||||
.boxed()
|
||||
.collect(Collectors.toMap(i -> ARR_LANGUAGES[i], i -> i)));
|
||||
public String[] strings = new String[ARR_LANGUAGES.length];
|
||||
|
||||
public TextStrings() {};
|
||||
|
||||
public TextStrings(String init) {
|
||||
for (int i = 0; i < NUM_LANGUAGES; i++)
|
||||
this.strings[i] = init;
|
||||
};
|
||||
|
||||
public TextStrings(List<String> strings, int key) {
|
||||
// Some hashes don't have strings for some languages :(
|
||||
String nullReplacement = "[N/A] %d".formatted((long) key & 0xFFFFFFFFL);
|
||||
for (int i = 0; i < NUM_LANGUAGES; i++) { // Find first non-null if there is any
|
||||
String s = strings.get(i);
|
||||
if (s != null) {
|
||||
nullReplacement = "[%s] - %s".formatted(ARR_LANGUAGES[i], s);
|
||||
break;
|
||||
}
|
||||
}
|
||||
for (int i = 0; i < NUM_LANGUAGES; i++) {
|
||||
String s = strings.get(i);
|
||||
if (s != null)
|
||||
this.strings[i] = s;
|
||||
else
|
||||
this.strings[i] = nullReplacement;
|
||||
}
|
||||
}
|
||||
|
||||
public String get(String languageCode) {
|
||||
return strings[MAP_LANGUAGES.getOrDefault(languageCode, 0)];
|
||||
}
|
||||
|
||||
public boolean set(String languageCode, String string) {
|
||||
int index = MAP_LANGUAGES.getOrDefault(languageCode, -1);
|
||||
if (index < 0) return false;
|
||||
strings[index] = string;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
private static final Pattern textMapKeyValueRegex = Pattern.compile("\"(\\d+)\": \"(.+)\"");
|
||||
|
||||
private static Int2ObjectMap<String> loadTextMapFile(String language, IntSet nameHashes) {
|
||||
Int2ObjectMap<String> output = new Int2ObjectOpenHashMap<>();
|
||||
try (BufferedReader file = new BufferedReader(new FileReader(Utils.toFilePath(RESOURCE("TextMap/TextMap"+language+".json")), StandardCharsets.UTF_8))) {
|
||||
Matcher matcher = textMapKeyValueRegex.matcher("");
|
||||
return new Int2ObjectOpenHashMap<>(
|
||||
file.lines()
|
||||
.sequential()
|
||||
.map(matcher::reset) // Side effects, but it's faster than making a new one
|
||||
.filter(Matcher::find)
|
||||
.filter(m -> nameHashes.contains((int) Long.parseLong(m.group(1)))) // TODO: Cache this parse somehow
|
||||
.collect(Collectors.toMap(
|
||||
m -> (int) Long.parseLong(m.group(1)),
|
||||
m -> m.group(2).replace("\\\"", "\""))));
|
||||
} catch (Exception e) {
|
||||
Grasscutter.getLogger().error("Error loading textmap: " + language);
|
||||
Grasscutter.getLogger().error(e.toString());
|
||||
}
|
||||
return output;
|
||||
}
|
||||
|
||||
private static Int2ObjectMap<TextStrings> loadTextMapFiles(IntSet nameHashes) {
|
||||
Map<Integer, Int2ObjectMap<String>> mapLanguageMaps = // Separate step to process the textmaps in parallel
|
||||
TextStrings.LIST_LANGUAGES.parallelStream().collect(
|
||||
Collectors.toConcurrentMap(s -> TextStrings.MAP_LANGUAGES.getInt(s), s -> loadTextMapFile(s, nameHashes)));
|
||||
List<Int2ObjectMap<String>> languageMaps =
|
||||
IntStream.range(0, TextStrings.NUM_LANGUAGES)
|
||||
.mapToObj(i -> mapLanguageMaps.get(i))
|
||||
.collect(Collectors.toList());
|
||||
|
||||
Map<TextStrings, TextStrings> canonicalTextStrings = new HashMap<>();
|
||||
return new Int2ObjectOpenHashMap<TextStrings>(
|
||||
nameHashes
|
||||
.intStream()
|
||||
.boxed()
|
||||
.collect(Collectors.toMap(key -> key, key -> {
|
||||
TextStrings t = new TextStrings(
|
||||
IntStream.range(0, TextStrings.NUM_LANGUAGES)
|
||||
.mapToObj(i -> languageMaps.get(i).get((int) key))
|
||||
.collect(Collectors.toList()), (int) key);
|
||||
return canonicalTextStrings.computeIfAbsent(t, x -> t);
|
||||
}))
|
||||
);
|
||||
}
|
||||
|
||||
private static Int2ObjectMap<TextStrings> loadTextMapsCache() throws Exception {
|
||||
try (ObjectInputStream file = new ObjectInputStream(new BufferedInputStream(Files.newInputStream(TEXTMAP_CACHE_PATH), 0x100000))) {
|
||||
final int fileVersion = file.readInt();
|
||||
if (fileVersion != TEXTMAP_CACHE_VERSION)
|
||||
throw new Exception("Invalid cache version");
|
||||
return (Int2ObjectMap<TextStrings>) file.readObject();
|
||||
}
|
||||
}
|
||||
|
||||
private static void saveTextMapsCache(Int2ObjectMap<TextStrings> input) throws IOException {
|
||||
try {
|
||||
Files.createDirectory(Path.of("cache"));
|
||||
} catch (FileAlreadyExistsException ignored) {};
|
||||
try (ObjectOutputStream file = new ObjectOutputStream(new BufferedOutputStream(Files.newOutputStream(TEXTMAP_CACHE_PATH, StandardOpenOption.CREATE), 0x100000))) {
|
||||
file.writeInt(TEXTMAP_CACHE_VERSION);
|
||||
file.writeObject(input);
|
||||
}
|
||||
}
|
||||
|
||||
private static Int2ObjectMap<TextStrings> textMapStrings;
|
||||
private static final Path TEXTMAP_CACHE_PATH = Path.of(Utils.toFilePath("cache/TextMapCache.bin"));
|
||||
|
||||
public static Int2ObjectMap<TextStrings> getTextMapStrings() {
|
||||
if (textMapStrings == null)
|
||||
loadTextMaps();
|
||||
return textMapStrings;
|
||||
}
|
||||
|
||||
public static TextStrings getTextMapKey(long hash) {
|
||||
return textMapStrings.get((int) hash);
|
||||
}
|
||||
|
||||
public static void loadTextMaps() {
|
||||
// Check system timestamps on cache and resources
|
||||
try {
|
||||
long cacheModified = Files.getLastModifiedTime(TEXTMAP_CACHE_PATH).toMillis();
|
||||
|
||||
long textmapsModified = Files.list(Path.of(RESOURCE("TextMap")))
|
||||
.filter(path -> path.toString().endsWith(".json"))
|
||||
.map(path -> {
|
||||
try {
|
||||
return Files.getLastModifiedTime(path).toMillis();
|
||||
} catch (Exception ignored) {
|
||||
Grasscutter.getLogger().debug("Exception while checking modified time: ", path);
|
||||
return Long.MAX_VALUE; // Don't use cache, something has gone wrong
|
||||
}
|
||||
})
|
||||
.max(Long::compare)
|
||||
.get();
|
||||
|
||||
Grasscutter.getLogger().debug("Cache modified %d, textmap modified %d".formatted(cacheModified, textmapsModified));
|
||||
if (textmapsModified < cacheModified) {
|
||||
// Try loading from cache
|
||||
Grasscutter.getLogger().info("Loading cached TextMaps");
|
||||
textMapStrings = loadTextMapsCache();
|
||||
return;
|
||||
}
|
||||
} catch (Exception e) {
|
||||
Grasscutter.getLogger().debug("Exception while checking cache: ", e);
|
||||
};
|
||||
|
||||
// Regenerate cache
|
||||
Grasscutter.getLogger().info("Generating TextMaps cache");
|
||||
ResourceLoader.loadAll();
|
||||
IntSet usedHashes = new IntOpenHashSet();
|
||||
GameData.getAvatarDataMap().forEach((k, v) -> usedHashes.add((int) v.getNameTextMapHash()));
|
||||
GameData.getItemDataMap().forEach((k, v) -> usedHashes.add((int) v.getNameTextMapHash()));
|
||||
GameData.getMonsterDataMap().forEach((k, v) -> usedHashes.add((int) v.getNameTextMapHash()));
|
||||
GameData.getMainQuestDataMap().forEach((k, v) -> usedHashes.add((int) v.getTitleTextMapHash()));
|
||||
GameData.getQuestDataMap().forEach((k, v) -> usedHashes.add((int) v.getDescTextMapHash()));
|
||||
// Incidental strings
|
||||
usedHashes.add((int) 4233146695L); // Character
|
||||
usedHashes.add((int) 4231343903L); // Weapon
|
||||
usedHashes.add((int) 332935371L); // Standard Wish
|
||||
usedHashes.add((int) 2272170627L); // Character Event Wish
|
||||
usedHashes.add((int) 3352513147L); // Character Event Wish-2
|
||||
usedHashes.add((int) 2864268523L); // Weapon Event Wish
|
||||
|
||||
textMapStrings = loadTextMapFiles(usedHashes);
|
||||
try {
|
||||
saveTextMapsCache(textMapStrings);
|
||||
} catch (IOException e) {
|
||||
Grasscutter.getLogger().error("Failed to save TextMap cache: ", e);
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user