Update delta/diff compression impl

server/channelserver/compression/deltacomp/deltacomp.go

@@ -0,0 +1,86 @@
+package deltacomp
+
+import (
+	"errors"
+
+	"github.com/Andoryuuta/byteframe"
+)
+
+func checkReadUint8(bf *byteframe.ByteFrame) (uint8, error) {
+	if len(bf.DataFromCurrent()) >= 1 {
+		return bf.ReadUint8(), nil
+	}
+	return 0, errors.New("Not enough data")
+}
+
+func checkReadUint16(bf *byteframe.ByteFrame) (uint16, error) {
+	if len(bf.DataFromCurrent()) >= 2 {
+		return bf.ReadUint16(), nil
+	}
+	return 0, errors.New("Not enough data")
+}
+
+func readCount(bf *byteframe.ByteFrame) (int, error) {
+	var count int
+
+	count8, err := checkReadUint8(bf)
+	if err != nil {
+		return 0, err
+	}
+	count = int(count8)
+
+	if count == 0 {
+		count16, err := checkReadUint16(bf)
+		if err != nil {
+			return 0, err
+		}
+		count = int(count16)
+	}
+
+	return int(count), nil
+}
+
+// ApplyDataDiff applies a delta data diff patch onto given base data.
+func ApplyDataDiff(diff []byte, baseData []byte) []byte {
+	// Make a copy of the base data to return,
+	// (probably just make this modify the given slice in the future).
+	baseCopy := make([]byte, len(baseData))
+	copy(baseCopy, baseData)
+
+	patch := byteframe.NewByteFrameFromBytes(diff)
+
+	// The very first matchCount is +1 more than it should be, so we start at -1.
+	dataOffset := -1
+	for {
+		// Read the amount of matching bytes.
+		matchCount, err := readCount(patch)
+		if err != nil {
+			// No more data
+			break
+		}
+
+		dataOffset += matchCount
+
+		// Read the amount of differing bytes.
+		differentCount, err := readCount(patch)
+		if err != nil {
+			// No more data
+			break
+		}
+		differentCount -= 1
+
+		// Apply the patch bytes.
+		for i := 0; i < differentCount; i++ {
+			b, err := checkReadUint8(patch)
+			if err != nil {
+				panic("Invalid or misunderstood patch format!")
+			}
+			baseCopy[dataOffset+i] = b
+		}
+
+		dataOffset += differentCount - 1
+
+	}
+
+	return baseCopy
+}

server/channelserver/compression/deltacomp/deltacomp_test.go

@@ -0,0 +1,113 @@
+package deltacomp
+
+import (
+	"bytes"
+	"encoding/hex"
+	"fmt"
+	"io/ioutil"
+	"testing"
+
+	"github.com/Andoryuuta/Erupe/server/channelserver/compression/nullcomp"
+)
+
+var tests = []struct {
+	before  string
+	patches []string
+	after   string
+}{
+	{
+		"hunternavi_0_before.bin",
+		[]string{
+			"hunternavi_0_patch_0.bin",
+			"hunternavi_0_patch_1.bin",
+		},
+		"hunternavi_0_after.bin",
+	},
+	{
+		// From "Character Progression 1 Creation-NPCs-Tours"
+		"hunternavi_1_before.bin",
+		[]string{
+			"hunternavi_1_patch_0.bin",
+			"hunternavi_1_patch_1.bin",
+			"hunternavi_1_patch_2.bin",
+			"hunternavi_1_patch_3.bin",
+			"hunternavi_1_patch_4.bin",
+			"hunternavi_1_patch_5.bin",
+			"hunternavi_1_patch_6.bin",
+			"hunternavi_1_patch_7.bin",
+			"hunternavi_1_patch_8.bin",
+			"hunternavi_1_patch_9.bin",
+			"hunternavi_1_patch_10.bin",
+			"hunternavi_1_patch_11.bin",
+			"hunternavi_1_patch_12.bin",
+			"hunternavi_1_patch_13.bin",
+			"hunternavi_1_patch_14.bin",
+			"hunternavi_1_patch_15.bin",
+			"hunternavi_1_patch_16.bin",
+			"hunternavi_1_patch_17.bin",
+			"hunternavi_1_patch_18.bin",
+			"hunternavi_1_patch_19.bin",
+			"hunternavi_1_patch_20.bin",
+			"hunternavi_1_patch_21.bin",
+			"hunternavi_1_patch_22.bin",
+			"hunternavi_1_patch_23.bin",
+			"hunternavi_1_patch_24.bin",
+		},
+		"hunternavi_1_after.bin",
+	},
+	{
+		// From "Progress Gogo GRP Grind 9 and Armor Upgrades and Partner Equip and Lost Cat and Manager talk and Pugi Order"
+		// Not really sure this one counts as a valid test as the input and output are exactly the same. The patches cancel each other out.
+		"platedata_0_before.bin",
+		[]string{
+			"platedata_0_patch_0.bin",
+			"platedata_0_patch_1.bin",
+		},
+		"platedata_0_after.bin",
+	},
+}
+
+func readTestDataFile(filename string) []byte {
+	data, err := ioutil.ReadFile(fmt.Sprintf("./test_data/%s", filename))
+	if err != nil {
+		panic(err)
+	}
+	return data
+}
+
+func TestDeltaPatch(t *testing.T) {
+	for k, tt := range tests {
+		testname := fmt.Sprintf("delta_patch_test_%d", k)
+		t.Run(testname, func(t *testing.T) {
+			// Load the test binary data.
+			beforeData, err := nullcomp.Decompress(readTestDataFile(tt.before))
+			if err != nil {
+				t.Error(err)
+			}
+
+			var patches [][]byte
+			for _, patchName := range tt.patches {
+				patchData := readTestDataFile(patchName)
+				patches = append(patches, patchData)
+			}
+
+			afterData, err := nullcomp.Decompress(readTestDataFile(tt.after))
+			if err != nil {
+				t.Error(err)
+			}
+
+			// Now actually test calling ApplyDataDiff.
+			data := beforeData
+
+			// Apply the patches in order.
+			for i, patch := range patches {
+				fmt.Println("patch index: ", i)
+				data = ApplyDataDiff(patch, data)
+			}
+
+			if !bytes.Equal(data, afterData) {
+				t.Errorf("got out\n\t%s\nwant\n\t%s", hex.Dump(data), hex.Dump(afterData))
+			}
+		})
+	}
+}

server/channelserver/handlers.go

@@ -13,6 +13,7 @@ import (
 	"time"
 
 	"github.com/Andoryuuta/Erupe/network/mhfpacket"
+	"github.com/Andoryuuta/Erupe/server/channelserver/compression/deltacomp"
 	"github.com/Andoryuuta/Erupe/server/channelserver/compression/nullcomp"
 	"github.com/Andoryuuta/byteframe"
 	"go.uber.org/zap"
@@ -53,41 +54,6 @@ func doSizedAckResp(s *Session, ackHandle uint32, data []byte) {
 	s.QueueAck(ackHandle, bfw.Data())
 }
 
-// process a datadiff save for platebox and platedata
-func saveDataDiff(b []byte, save []byte) []byte {
-	// there are a bunch of extra variations on this method in use which this does not handle yet
-	// specifically this is for diffs with seek amounts trailed by 02 followed by bytes to be written
-	var seekBytes []byte
-	seekOperation := 0
-	write := byte(0)
-	for len(b) > 2 {
-		if bytes.IndexRune(b, 2) != 0 {
-			seekBytes = b[:bytes.IndexRune(b, 2)+1]
-		} else {
-			seekBytes = b[:bytes.IndexRune(b[1:], 2)+2]
-		}
-		if len(seekBytes) == 1 {
-			seekBytes = b[:bytes.IndexRune(b, 2)+2]
-			//fmt.Printf("Seek: %d SeekBytes: %X Write: %X\n", seekBytes[0], seekBytes, b[len(seekBytes)] )
-			seekOperation += int(seekBytes[0])
-			write = b[len(seekBytes)]
-			b = b[3:]
-		} else {
-			seek := int32(0)
-			for _, b := range seekBytes[:len(seekBytes)-1] {
-				seek = (seek << 8) | int32(b)
-			}
-			//fmt.Printf("Seek: %d SeekBytes: %X Write: %X\n", seek, seekBytes, b[len(seekBytes)] )
-			seekOperation += int(seek)
-			write = b[len(seekBytes)]
-			b = b[len(seekBytes)+1:]
-		}
-		save[seekOperation-1] = write
-	}
-
-	return save
-}
-
 func updateRights(s *Session) {
 	update := &mhfpacket.MsgSysUpdateRight{
 		Unk0: 0,
@@ -930,28 +896,80 @@ func handleMsgSysReserve5F(s *Session, p mhfpacket.MHFPacket) {}
 
 func handleMsgMhfSavedata(s *Session, p mhfpacket.MHFPacket) {
 	pkt := p.(*mhfpacket.MsgMhfSavedata)
+
 	err := ioutil.WriteFile(fmt.Sprintf("savedata\\%d.bin", time.Now().Unix()), pkt.RawDataPayload, 0644)
 	if err != nil {
 		s.logger.Fatal("Error dumping savedata", zap.Error(err))
 	}
-	if pkt.SaveType == 2 {
-		_, err = s.server.db.Exec("UPDATE characters SET is_new_character=false, savedata=$1 WHERE id=$2", pkt.RawDataPayload, s.charID)
 
-		// Temporary server launcher response stuff
-		// 0x1F715	Weapon Class
-		// 0x1FDF6 HR (small_gr_level)
-		// 0x88 Character Name
-		saveFile, _ := nullcomp.Decompress(pkt.RawDataPayload)
-		_, err = s.server.db.Exec("UPDATE characters SET weapon=$1 WHERE id=$2", uint16(saveFile[128789]), s.charID)
-		x := uint16(saveFile[130550])<<8 | uint16(saveFile[130551])
-		_, err = s.server.db.Exec("UPDATE characters SET small_gr_level=$1 WHERE id=$2", uint16(x), s.charID)
-		_, err = s.server.db.Exec("UPDATE characters SET name=$1 WHERE id=$2", strings.SplitN(string(saveFile[88:100]), "\x00", 2)[0], s.charID)
+	// Var to hold the decompressed savedata for updating the launcher response fields.
+	var decompressedData []byte
+
+	if pkt.SaveType == 1 {
+		// Diff-based update.
+
+		// Load existing save
+		var data []byte
+		err := s.server.db.QueryRow("SELECT savedata FROM characters WHERE id = $1", s.charID).Scan(&data)
+		if err != nil {
+			s.logger.Fatal("Failed to get savedata from db", zap.Error(err))
+		}
+
+		// Decompress
+		s.logger.Info("Decompressing...")
+		data, err = nullcomp.Decompress(data)
+		if err != nil {
+			s.logger.Fatal("Failed to decompress savedata from db", zap.Error(err))
+		}
+
+		// Perform diff and compress it to write back to db
+		s.logger.Info("Diffing...")
+		saveOutput, err := nullcomp.Compress(deltacomp.ApplyDataDiff(pkt.RawDataPayload, data))
+		if err != nil {
+			s.logger.Fatal("Failed to diff and compress savedata", zap.Error(err))
+		}
+
+		decompressedData = saveOutput // For updating launcher fields.
+
+		_, err = s.server.db.Exec("UPDATE characters SET savedata=$1 WHERE id=$2", saveOutput, s.charID)
+		if err != nil {
+			s.logger.Fatal("Failed to update savedata in db", zap.Error(err))
+		}
+
+		s.logger.Info("Wrote recompressed savedata back to DB.")
+	} else {
+		// Regular blob update.
+
+		_, err = s.server.db.Exec("UPDATE characters SET is_new_character=false, savedata=$1 WHERE id=$2", pkt.RawDataPayload, s.charID)
 
 		if err != nil {
 			s.logger.Fatal("Failed to update savedata in db", zap.Error(err))
 		}
-	} else {
-		fmt.Printf("Got savedata packet of type 1, no handling implemented. Not saving.")
+
+		decompressedData, err = nullcomp.Decompress(pkt.RawDataPayload) // For updating launcher fields.
+		if err != nil {
+			s.logger.Fatal("Failed to decompress savedata from packet", zap.Error(err))
+		}
+	}
+
+	// Temporary server launcher response stuff
+	// 0x1F715	Weapon Class
+	// 0x1FDF6 HR (small_gr_level)
+	// 0x88 Character Name
+	_, err = s.server.db.Exec("UPDATE characters SET weapon=$1 WHERE id=$2", uint16(decompressedData[128789]), s.charID)
+	if err != nil {
+		s.logger.Fatal("Failed to character weapon in db", zap.Error(err))
+	}
+
+	x := uint16(decompressedData[130550])<<8 | uint16(decompressedData[130551])
+	_, err = s.server.db.Exec("UPDATE characters SET small_gr_level=$1 WHERE id=$2", uint16(x), s.charID)
+	if err != nil {
+		s.logger.Fatal("Failed to character small_gr_level in db", zap.Error(err))
+	}
+
+	_, err = s.server.db.Exec("UPDATE characters SET name=$1 WHERE id=$2", strings.SplitN(string(decompressedData[88:100]), "\x00", 2)[0], s.charID)
+	if err != nil {
+		s.logger.Fatal("Failed to character name in db", zap.Error(err))
 	}
 
 	s.QueueAck(pkt.AckHandle, []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00})
@@ -1532,34 +1550,41 @@ func handleMsgMhfLoadPlateData(s *Session, p mhfpacket.MHFPacket) {
 
 func handleMsgMhfSavePlateData(s *Session, p mhfpacket.MHFPacket) {
 	pkt := p.(*mhfpacket.MsgMhfSavePlateData)
+
 	err := ioutil.WriteFile(fmt.Sprintf("savedata\\%d_platedata.bin", time.Now().Unix()), pkt.RawDataPayload, 0644)
 	if err != nil {
 		s.logger.Fatal("Error dumping platedata", zap.Error(err))
 	}
+
 	if pkt.IsDataDiff {
-		// https://gist.github.com/Andoryuuta/9c524da7285e4b5ca7e52e0fc1ca1daf
 		var data []byte
-		//load existing save
+
+		// Load existing save
 		err := s.server.db.QueryRow("SELECT platedata FROM characters WHERE id = $1", s.charID).Scan(&data)
 		if err != nil {
 			s.logger.Fatal("Failed to get platedata savedata from db", zap.Error(err))
 		}
-		//decompress
-		fmt.Println("Decompressing...")
+
+		// Decompress
+		s.logger.Info("Decompressing...")
 		data, err = nullcomp.Decompress(data)
 		if err != nil {
 			s.logger.Fatal("Failed to decompress platedata from db", zap.Error(err))
 		}
-		// perform diff and compress it to write back to db
-		fmt.Println("Diffing...")
-		saveOutput, err := nullcomp.Compress(saveDataDiff(pkt.RawDataPayload, data))
+
+		// Perform diff and compress it to write back to db
+		s.logger.Info("Diffing...")
+		saveOutput, err := nullcomp.Compress(deltacomp.ApplyDataDiff(pkt.RawDataPayload, data))
 		if err != nil {
 			s.logger.Fatal("Failed to diff and compress platedata savedata", zap.Error(err))
 		}
+
 		_, err = s.server.db.Exec("UPDATE characters SET platedata=$1 WHERE id=$2", saveOutput, s.charID)
 		if err != nil {
 			s.logger.Fatal("Failed to update platedata savedata in db", zap.Error(err))
 		}
+
+		s.logger.Info("Wrote recompressed platedata back to DB.")
 	} else {
 		// simply update database, no extra processing
 		_, err := s.server.db.Exec("UPDATE characters SET platedata=$1 WHERE id=$2", pkt.RawDataPayload, s.charID)
@@ -1567,6 +1592,7 @@ func handleMsgMhfSavePlateData(s *Session, p mhfpacket.MHFPacket) {
 			s.logger.Fatal("Failed to update platedata savedata in db", zap.Error(err))
 		}
 	}
+
 	s.QueueAck(pkt.AckHandle, []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00})
 }
 
@@ -1587,36 +1613,41 @@ func handleMsgMhfLoadPlateBox(s *Session, p mhfpacket.MHFPacket) {
 
 func handleMsgMhfSavePlateBox(s *Session, p mhfpacket.MHFPacket) {
 	pkt := p.(*mhfpacket.MsgMhfSavePlateBox)
+
 	err := ioutil.WriteFile(fmt.Sprintf("savedata\\%d_platebox.bin", time.Now().Unix()), pkt.RawDataPayload, 0644)
 	if err != nil {
 		s.logger.Fatal("Error dumping hunter platebox savedata", zap.Error(err))
 	}
+
 	if pkt.IsDataDiff {
 		var data []byte
-		//load existing save
+
+		// Load existing save
 		err := s.server.db.QueryRow("SELECT platebox FROM characters WHERE id = $1", s.charID).Scan(&data)
 		if err != nil {
 			s.logger.Fatal("Failed to get sigil box savedata from db", zap.Error(err))
 		}
-		//decompress
-		fmt.Println("Decompressing...")
+
+		// Decompress
+		s.logger.Info("Decompressing...")
 		data, err = nullcomp.Decompress(data)
 		if err != nil {
 			s.logger.Fatal("Failed to decompress savedata from db", zap.Error(err))
 		}
-		// perform diff and compress it to write back to db
-		fmt.Println("Diffing...")
-		saveOutput, err := nullcomp.Compress(saveDataDiff(pkt.RawDataPayload, data))
+
+		// Perform diff and compress it to write back to db
+		s.logger.Info("Diffing...")
+		saveOutput, err := nullcomp.Compress(deltacomp.ApplyDataDiff(pkt.RawDataPayload, data))
 		if err != nil {
 			s.logger.Fatal("Failed to diff and compress savedata", zap.Error(err))
 		}
+
 		_, err = s.server.db.Exec("UPDATE characters SET platebox=$1 WHERE id=$2", saveOutput, s.charID)
 		if err != nil {
 			s.logger.Fatal("Failed to update platebox savedata in db", zap.Error(err))
-		} else {
-			fmt.Println("Wrote recompressed save back to DB.")
 		}
 
+		s.logger.Info("Wrote recompressed platebox back to DB.")
 	} else {
 		// simply update database, no extra processing
 		_, err := s.server.db.Exec("UPDATE characters SET platebox=$1 WHERE id=$2", pkt.RawDataPayload, s.charID)
@@ -1911,8 +1942,34 @@ func handleMsgMhfSaveHunterNavi(s *Session, p mhfpacket.MHFPacket) {
 	}
 
 	if pkt.IsDataDiff {
-		// https://gist.github.com/Andoryuuta/9c524da7285e4b5ca7e52e0fc1ca1daf
-		// doesn't seem fully consistent with platedata?
+		var data []byte
+
+		// Load existing save
+		err := s.server.db.QueryRow("SELECT hunternavi FROM characters WHERE id = $1", s.charID).Scan(&data)
+		if err != nil {
+			s.logger.Fatal("Failed to get hunternavi savedata from db", zap.Error(err))
+		}
+
+		// Decompress
+		s.logger.Info("Decompressing...")
+		data, err = nullcomp.Decompress(data)
+		if err != nil {
+			s.logger.Fatal("Failed to decompress hunternavi from db", zap.Error(err))
+		}
+
+		// Perform diff and compress it to write back to db
+		s.logger.Info("Diffing...")
+		saveOutput, err := nullcomp.Compress(deltacomp.ApplyDataDiff(pkt.RawDataPayload, data))
+		if err != nil {
+			s.logger.Fatal("Failed to diff and compress hunternavi savedata", zap.Error(err))
+		}
+
+		_, err = s.server.db.Exec("UPDATE characters SET hunternavi=$1 WHERE id=$2", saveOutput, s.charID)
+		if err != nil {
+			s.logger.Fatal("Failed to update hunternavi savedata in db", zap.Error(err))
+		}
+
+		s.logger.Info("Wrote recompressed hunternavi back to DB.")
 	} else {
 		// simply update database, no extra processing
 		_, err := s.server.db.Exec("UPDATE characters SET hunternavi=$1 WHERE id=$2", pkt.RawDataPayload, s.charID)