repository cleanup

server/channelserver/compression/deltacomp/deltacomp.go

@@ -0,0 +1,105 @@
+package deltacomp
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+)
+
+func checkReadUint8(r *bytes.Reader) (uint8, error) {
+	b, err := r.ReadByte()
+	if err != nil {
+		return 0, err
+	}
+	return b, nil
+}
+
+func checkReadUint16(r *bytes.Reader) (uint16, error) {
+	data := make([]byte, 2)
+	n, err := r.Read(data)
+	if err != nil {
+		return 0, err
+	} else if n != len(data) {
+		return 0, io.EOF
+	}
+
+	return uint16(data[0])<<8 | uint16(data[1]), nil
+}
+
+func readCount(r *bytes.Reader) (int, error) {
+	var count int
+
+	count8, err := checkReadUint8(r)
+	if err != nil {
+		return 0, err
+	}
+	count = int(count8)
+
+	if count == 0 {
+		count16, err := checkReadUint16(r)
+		if err != nil {
+			return 0, err
+		}
+		count = int(count16)
+	}
+
+	return int(count), nil
+}
+
+// ApplyDataDiff applies a delta data diff patch onto given base data.
+func ApplyDataDiff(diff []byte, baseData []byte) []byte {
+	// Make a copy of the base data to return,
+	// (probably just make this modify the given slice in the future).
+	baseCopy := make([]byte, len(baseData))
+	copy(baseCopy, baseData)
+
+	patch := bytes.NewReader(diff)
+
+	// The very first matchCount is +1 more than it should be, so we start at -1.
+	dataOffset := -1
+	for {
+		// Read the amount of matching bytes.
+		matchCount, err := readCount(patch)
+		if err != nil {
+			// No more data
+			break
+		}
+
+		dataOffset += matchCount
+
+		// Read the amount of differing bytes.
+		differentCount, err := readCount(patch)
+		if err != nil {
+			// No more data
+			break
+		}
+		differentCount--
+
+		// Grow slice if it's required
+		if len(baseCopy) < dataOffset {
+			fmt.Printf("Slice smaller than data offset, growing slice...")
+			baseCopy = append(baseCopy, make([]byte, (dataOffset+differentCount)-len(baseData))...)
+		} else {
+			length := len(baseCopy[dataOffset:])
+			if length < differentCount {
+				length -= differentCount
+				baseCopy = append(baseCopy, make([]byte, length)...)
+			}
+		}
+
+		// Apply the patch bytes.
+		for i := 0; i < differentCount; i++ {
+			b, err := checkReadUint8(patch)
+			if err != nil {
+				panic("Invalid or misunderstood patch format!")
+			}
+
+			baseCopy[dataOffset+i] = b
+		}
+
+		dataOffset += differentCount - 1
+
+	}
+
+	return baseCopy
+}

server/channelserver/compression/deltacomp/deltacomp_test.go

@@ -0,0 +1,113 @@
+package deltacomp
+
+import (
+	"bytes"
+	"encoding/hex"
+	"fmt"
+	"io/ioutil"
+	"testing"
+
+	"erupe-ce/server/channelserver/compression/nullcomp"
+)
+
+var tests = []struct {
+	before  string
+	patches []string
+	after   string
+}{
+	{
+		"hunternavi_0_before.bin",
+		[]string{
+			"hunternavi_0_patch_0.bin",
+			"hunternavi_0_patch_1.bin",
+		},
+		"hunternavi_0_after.bin",
+	},
+	{
+		// From "Character Progression 1 Creation-NPCs-Tours"
+		"hunternavi_1_before.bin",
+		[]string{
+			"hunternavi_1_patch_0.bin",
+			"hunternavi_1_patch_1.bin",
+			"hunternavi_1_patch_2.bin",
+			"hunternavi_1_patch_3.bin",
+			"hunternavi_1_patch_4.bin",
+			"hunternavi_1_patch_5.bin",
+			"hunternavi_1_patch_6.bin",
+			"hunternavi_1_patch_7.bin",
+			"hunternavi_1_patch_8.bin",
+			"hunternavi_1_patch_9.bin",
+			"hunternavi_1_patch_10.bin",
+			"hunternavi_1_patch_11.bin",
+			"hunternavi_1_patch_12.bin",
+			"hunternavi_1_patch_13.bin",
+			"hunternavi_1_patch_14.bin",
+			"hunternavi_1_patch_15.bin",
+			"hunternavi_1_patch_16.bin",
+			"hunternavi_1_patch_17.bin",
+			"hunternavi_1_patch_18.bin",
+			"hunternavi_1_patch_19.bin",
+			"hunternavi_1_patch_20.bin",
+			"hunternavi_1_patch_21.bin",
+			"hunternavi_1_patch_22.bin",
+			"hunternavi_1_patch_23.bin",
+			"hunternavi_1_patch_24.bin",
+		},
+		"hunternavi_1_after.bin",
+	},
+	{
+		// From "Progress Gogo GRP Grind 9 and Armor Upgrades and Partner Equip and Lost Cat and Manager talk and Pugi Order"
+		// Not really sure this one counts as a valid test as the input and output are exactly the same. The patches cancel each other out.
+		"platedata_0_before.bin",
+		[]string{
+			"platedata_0_patch_0.bin",
+			"platedata_0_patch_1.bin",
+		},
+		"platedata_0_after.bin",
+	},
+}
+
+func readTestDataFile(filename string) []byte {
+	data, err := ioutil.ReadFile(fmt.Sprintf("./test_data/%s", filename))
+	if err != nil {
+		panic(err)
+	}
+	return data
+}
+
+func TestDeltaPatch(t *testing.T) {
+	for k, tt := range tests {
+		testname := fmt.Sprintf("delta_patch_test_%d", k)
+		t.Run(testname, func(t *testing.T) {
+			// Load the test binary data.
+			beforeData, err := nullcomp.Decompress(readTestDataFile(tt.before))
+			if err != nil {
+				t.Error(err)
+			}
+
+			var patches [][]byte
+			for _, patchName := range tt.patches {
+				patchData := readTestDataFile(patchName)
+				patches = append(patches, patchData)
+			}
+
+			afterData, err := nullcomp.Decompress(readTestDataFile(tt.after))
+			if err != nil {
+				t.Error(err)
+			}
+
+			// Now actually test calling ApplyDataDiff.
+			data := beforeData
+
+			// Apply the patches in order.
+			for i, patch := range patches {
+				fmt.Println("patch index: ", i)
+				data = ApplyDataDiff(patch, data)
+			}
+
+			if !bytes.Equal(data, afterData) {
+				t.Errorf("got out\n\t%s\nwant\n\t%s", hex.Dump(data), hex.Dump(afterData))
+			}
+		})
+	}
+}

server/channelserver/compression/nullcomp/nullcomp.go

@@ -0,0 +1,98 @@
+package nullcomp
+
+import (
+	"bytes"
+	"io"
+)
+
+// Decompress decompresses null-compressesed data.
+func Decompress(compData []byte) ([]byte, error) {
+	r := bytes.NewReader(compData)
+
+	header := make([]byte, 16)
+	n, err := r.Read(header)
+	if err != nil {
+		return nil, err
+	} else if n != len(header) {
+		return nil, err
+	}
+
+	// Just return the data if it doesn't contain the cmp header.
+	if !bytes.Equal(header, []byte("cmp\x2020110113\x20\x20\x20\x00")) {
+		return compData, nil
+	}
+
+	var output []byte
+	for {
+		b, err := r.ReadByte()
+		if err == io.EOF {
+			break
+		} else if err != nil {
+			return nil, err
+		}
+
+		if b == 0 {
+			// If it's a null byte, then the next byte is how many nulls to add.
+			nullCount, err := r.ReadByte()
+			if err == io.EOF {
+				break
+			} else if err != nil {
+				return nil, err
+			}
+
+			output = append(output, make([]byte, int(nullCount))...)
+		} else {
+			output = append(output, b)
+		}
+	}
+
+	return output, nil
+}
+
+// Compress null compresses give given data.
+func Compress(rawData []byte) ([]byte, error) {
+	r := bytes.NewReader(rawData)
+	var output []byte
+	output = append(output, []byte("cmp\x2020110113\x20\x20\x20\x00")...)
+	for {
+		b, err := r.ReadByte()
+		if err == io.EOF {
+			break
+		} else if err != nil {
+			return nil, err
+		}
+		if b == 0 {
+			output = append(output, []byte{0x00}...)
+			// read to get null count
+			nullCount := 1
+			for {
+				i, err := r.ReadByte()
+				if err == io.EOF {
+					output = append(output, []byte{byte(nullCount)}...)
+					break
+				} else if i != 0 && nullCount != 0 {
+					r.UnreadByte()
+					output = append(output, []byte{byte(nullCount)}...)
+					break
+				} else if i != 0 && nullCount == 0 {
+					r.UnreadByte()
+					output = output[:len(output)-2]
+					output = append(output, []byte{byte(0xFF)}...)
+					break
+				} else if err != nil {
+					return nil, err
+				}
+				nullCount++
+
+				// Flush the null-count if it gets to 255, start on the next null count.
+				if nullCount == 255 {
+					output = append(output, []byte{0xFF, 0x00}...)
+					nullCount = 0
+				}
+			}
+		} else {
+			output = append(output, b)
+		}
+	}
+	return output, nil
+}