feat(scenario): add JSON scenario support and JKR type-3 compressor

Closes #172. Scenario files in bin/scenarios/ can now be authored as
.json instead of .bin — the server compiles them to wire format on
load, falling back to .bin if no .json is present.

- Add ParseScenarioBinary / CompileScenarioJSON in scenario_json.go;
  supports sub-header format (strings as UTF-8, metadata as base64),
  inline format, and raw JKR blobs.
- Add PackSimple JKR type-3 (LZ77) compressor in jpk_compress.go,
  ported from ReFrontier JPKEncodeLz.cs; round-trip tested against
  UnpackSimple.
- Fix off-by-one in processDecode (jpk.go): last literal byte was
  silently dropped for data that does not end on a back-reference.
- Wire loadScenarioBinary into handleMsgSysGetFile replacing the
  inline os.ReadFile call; mirrors the existing loadQuestBinary pattern.
- Rewrite docs/scenario-format.md with full container/sub-header spec
  and JSON schema examples.
This commit is contained in:
Houmgaor
2026-03-20 13:55:40 +01:00
parent 71b675bf3e
commit a1dfdd330a
8 changed files with 1226 additions and 27 deletions

View File

@@ -54,7 +54,7 @@ func ProcessDecode(data *byteframe.ByteFrame, outBuffer []byte) {
func (s *jpkState) processDecode(data *byteframe.ByteFrame, outBuffer []byte) {
outIndex := 0
for int(data.Index()) < len(data.Data()) && outIndex < len(outBuffer)-1 {
for int(data.Index()) < len(data.Data()) && outIndex < len(outBuffer) {
if s.bitShift(data) == 0 {
outBuffer[outIndex] = ReadByte(data)
outIndex++

View File

@@ -0,0 +1,169 @@
package decryption
import "encoding/binary"
// PackSimple compresses data using JPK type-3 (LZ77) compression and wraps it
// in a JKR header. It is the inverse of UnpackSimple.
func PackSimple(data []byte) []byte {
compressed := lzEncode(data)
out := make([]byte, 16+len(compressed))
binary.LittleEndian.PutUint32(out[0:4], 0x1A524B4A) // JKR magic
binary.LittleEndian.PutUint16(out[4:6], 0x0108) // version
binary.LittleEndian.PutUint16(out[6:8], 0x0003) // type 3 = LZ only
binary.LittleEndian.PutUint32(out[8:12], 0x00000010) // data offset = 16 (after header)
binary.LittleEndian.PutUint32(out[12:16], uint32(len(data)))
copy(out[16:], compressed)
return out
}
// lzEncoder holds mutable state for the LZ77 compression loop.
// Ported from ReFrontier JPKEncodeLz.cs.
//
// The format groups 8 items behind a flag byte (MSB = item 0):
//
// bit=0 → literal byte follows
// bit=1 → back-reference follows (with sub-cases below)
//
// Back-reference sub-cases:
//
// 10xx + 1 byte → length 36, offset ≤ 255
// 11 + 2 bytes → length 39, offset ≤ 8191 (length encoded in hi byte bits 75)
// 11 + 2 bytes + 0 + 4 bits → length 1025, offset ≤ 8191
// 11 + 2 bytes + 1 + 1 byte → length 26280, offset ≤ 8191
type lzEncoder struct {
flag byte
shiftIndex int
toWrite [1024]byte // data bytes for the current flag group
indexToWrite int
out []byte
}
func (e *lzEncoder) setFlag(value bool) {
if e.shiftIndex <= 0 {
e.flushFlag(false)
e.shiftIndex = 7
} else {
e.shiftIndex--
}
if value {
e.flag |= 1 << uint(e.shiftIndex)
}
}
// setFlagsReverse writes `count` bits of value MSB-first.
func (e *lzEncoder) setFlagsReverse(value byte, count int) {
for i := count - 1; i >= 0; i-- {
e.setFlag(((value >> uint(i)) & 1) == 1)
}
}
func (e *lzEncoder) writeByte(b byte) {
e.toWrite[e.indexToWrite] = b
e.indexToWrite++
}
func (e *lzEncoder) flushFlag(final bool) {
if !final || e.indexToWrite > 0 {
e.out = append(e.out, e.flag)
}
e.flag = 0
e.out = append(e.out, e.toWrite[:e.indexToWrite]...)
e.indexToWrite = 0
}
// lzEncode compresses data with the JPK LZ77 algorithm, producing the raw
// compressed bytes (without the JKR header).
func lzEncode(data []byte) []byte {
const (
compressionLevel = 280 // max match length
maxIndexDist = 0x300 // max look-back distance (768)
)
enc := &lzEncoder{shiftIndex: 8}
for pos := 0; pos < len(data); {
repLen, repOff := lzLongestRepetition(data, pos, compressionLevel, maxIndexDist)
if repLen == 0 {
// Literal byte
enc.setFlag(false)
enc.writeByte(data[pos])
pos++
} else {
enc.setFlag(true)
if repLen <= 6 && repOff <= 0xff {
// Short: flag=10, 2-bit length, 1-byte offset
enc.setFlag(false)
enc.setFlagsReverse(byte(repLen-3), 2)
enc.writeByte(byte(repOff))
} else {
// Long: flag=11, 2-byte offset/length header
enc.setFlag(true)
u16 := uint16(repOff)
if repLen <= 9 {
// Length fits in hi byte bits 7-5
u16 |= uint16(repLen-2) << 13
}
enc.writeByte(byte(u16 >> 8))
enc.writeByte(byte(u16 & 0xff))
if repLen > 9 {
if repLen <= 25 {
// Extended: flag=0, 4-bit length
enc.setFlag(false)
enc.setFlagsReverse(byte(repLen-10), 4)
} else {
// Extended: flag=1, 1-byte length
enc.setFlag(true)
enc.writeByte(byte(repLen - 0x1a))
}
}
}
pos += repLen
}
}
enc.flushFlag(true)
return enc.out
}
// lzLongestRepetition finds the longest match for data[pos:] in the look-back
// window. Returns (matchLen, encodedOffset) where encodedOffset is
// (pos - matchStart - 1). Returns (0, 0) when no usable match exists.
func lzLongestRepetition(data []byte, pos, compressionLevel, maxIndexDist int) (int, uint) {
const minLength = 3
// Clamp threshold to available bytes
threshold := compressionLevel
if remaining := len(data) - pos; remaining < threshold {
threshold = remaining
}
if pos == 0 || threshold < minLength {
return 0, 0
}
windowStart := pos - maxIndexDist
if windowStart < 0 {
windowStart = 0
}
maxLen := 0
var bestOffset uint
for left := windowStart; left < pos; left++ {
curLen := 0
for curLen < threshold && data[left+curLen] == data[pos+curLen] {
curLen++
}
if curLen >= minLength && curLen > maxLen {
maxLen = curLen
bestOffset = uint(pos - left - 1)
if maxLen >= threshold {
break
}
}
}
return maxLen, bestOffset
}

View File

@@ -0,0 +1,78 @@
package decryption
import (
"bytes"
"encoding/binary"
"testing"
)
func TestPackSimpleRoundTrip(t *testing.T) {
tests := []struct {
name string
data []byte
}{
{"single byte", []byte{0x42}},
{"ascii text", []byte("hello world")},
{"repeated pattern", bytes.Repeat([]byte{0xAB, 0xCD}, 100)},
{"all zeros", make([]byte, 256)},
{"all 0xFF", bytes.Repeat([]byte{0xFF}, 128)},
{"sequential bytes", func() []byte {
b := make([]byte, 256)
for i := range b {
b[i] = byte(i)
}
return b
}()},
{"long repeating run", bytes.Repeat([]byte("ABCDEFGH"), 50)},
{"mixed", []byte{0x00, 0x01, 0x02, 0xFF, 0xFE, 0xFD, 0x80, 0x81}},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
compressed := PackSimple(tc.data)
got := UnpackSimple(compressed)
if !bytes.Equal(got, tc.data) {
t.Errorf("round-trip mismatch\n want len=%d\n got len=%d", len(tc.data), len(got))
}
})
}
}
func TestPackSimpleHeader(t *testing.T) {
data := []byte("test data")
compressed := PackSimple(data)
if len(compressed) < 16 {
t.Fatalf("output too short: %d bytes", len(compressed))
}
magic := binary.LittleEndian.Uint32(compressed[0:4])
if magic != 0x1A524B4A {
t.Errorf("wrong magic: got 0x%08X, want 0x1A524B4A", magic)
}
jpkType := binary.LittleEndian.Uint16(compressed[6:8])
if jpkType != 3 {
t.Errorf("wrong type: got %d, want 3", jpkType)
}
decompSize := binary.LittleEndian.Uint32(compressed[12:16])
if decompSize != uint32(len(data)) {
t.Errorf("wrong decompressed size: got %d, want %d", decompSize, len(data))
}
}
func TestPackSimpleLargeRepeating(t *testing.T) {
// 4 KB of repeating pattern — should compress well
data := bytes.Repeat([]byte{0xAA, 0xBB, 0xCC, 0xDD}, 1024)
compressed := PackSimple(data)
if len(compressed) >= len(data) {
t.Logf("note: compressed (%d) not smaller than original (%d)", len(compressed), len(data))
}
got := UnpackSimple(compressed)
if !bytes.Equal(got, data) {
t.Errorf("round-trip failed for large repeating data")
}
}