BackupX/server/internal/backup/repository_chunker.go

package backup

import (
	"context"
	"fmt"
	"io"
)

const (
	repositoryChunkMin = 512 << 10
	repositoryChunkAvg = 1 << 20
	repositoryChunkMax = 4 << 20
)

// contentDefinedChunker implements the normalized FastCDC cut-point strategy.
// The rolling Gear hash only retains the latest 64 bytes through uint64
// overflow, so chunk boundaries re-synchronize after insertions or deletions.
type contentDefinedChunker struct {
	minSize   int
	avgSize   int
	maxSize   int
	smallMask uint64
	largeMask uint64
	gear      [256]uint64
}

func newContentDefinedChunker() *contentDefinedChunker {
	chunker := &contentDefinedChunker{
		minSize:   repositoryChunkMin,
		avgSize:   repositoryChunkAvg,
		maxSize:   repositoryChunkMax,
		smallMask: (1 << 21) - 1,
		largeMask: (1 << 19) - 1,
	}

	// SplitMix64 produces a stable, well-distributed Gear table. The seed and
	// generation algorithm are part of repository format v1 and must not change.
	seed := uint64(0x6a09e667f3bcc909)
	for i := range chunker.gear {
		seed += 0x9e3779b97f4a7c15
		value := seed
		value = (value ^ (value >> 30)) * 0xbf58476d1ce4e5b9
		value = (value ^ (value >> 27)) * 0x94d049bb133111eb
		chunker.gear[i] = value ^ (value >> 31)
	}
	return chunker
}

func (c *contentDefinedChunker) Split(ctx context.Context, reader io.Reader, emit func([]byte) error) error {
	if reader == nil || emit == nil {
		return fmt.Errorf("chunk reader and emitter are required")
	}

	pending := make([]byte, 0, c.maxSize+(256<<10))
	readBuffer := make([]byte, 256<<10)
	eof := false
	for {
		if !eof {
			readCount, readErr := reader.Read(readBuffer)
			if readCount > 0 {
				pending = append(pending, readBuffer[:readCount]...)
			}
			switch readErr {
			case nil:
			case io.EOF:
				eof = true
			default:
				return fmt.Errorf("read source for chunking: %w", readErr)
			}
			if readCount == 0 && readErr == nil {
				continue
			}
		}

		for len(pending) > 0 {
			if err := ctx.Err(); err != nil {
				return err
			}
			cut := c.findCutPoint(pending, eof)
			if cut == 0 {
				break
			}
			chunk := make([]byte, cut)
			copy(chunk, pending[:cut])
			if err := emit(chunk); err != nil {
				return err
			}
			copy(pending, pending[cut:])
			pending = pending[:len(pending)-cut]
		}

		if eof {
			if len(pending) != 0 {
				return fmt.Errorf("chunker stopped with %d buffered bytes", len(pending))
			}
			return nil
		}
	}
}

func (c *contentDefinedChunker) findCutPoint(data []byte, eof bool) int {
	if len(data) < c.minSize {
		if eof {
			return len(data)
		}
		return 0
	}

	limit := len(data)
	if limit > c.maxSize {
		limit = c.maxSize
	}
	var hash uint64
	for index := c.minSize; index < limit; index++ {
		hash = (hash << 1) + c.gear[data[index]]
		mask := c.largeMask
		if index < c.avgSize {
			mask = c.smallMask
		}
		if hash&mask == 0 {
			return index + 1
		}
	}
	if len(data) >= c.maxSize {
		return c.maxSize
	}
	if eof {
		return len(data)
	}
	return 0
}