diff --git a/go.mod b/go.mod index 4eeadea..31d976c 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.23.0 require ( filippo.io/age v1.2.1 - github.com/andybalholm/brotli v1.1.1 + github.com/andybalholm/brotli v1.2.0 github.com/cloudflare/circl v1.6.0 github.com/klauspost/compress v1.18.0 github.com/stretchr/testify v1.10.0 diff --git a/go.sum b/go.sum index 47281df..74ec298 100644 --- a/go.sum +++ b/go.sum @@ -2,8 +2,8 @@ c2sp.org/CCTV/age v0.0.0-20240306222714-3ec4d716e805 h1:u2qwJeEvnypw+OCPUHmoZE3I c2sp.org/CCTV/age v0.0.0-20240306222714-3ec4d716e805/go.mod h1:FomMrUJ2Lxt5jCLmZkG3FHa72zUprnhd3v/Z18Snm4w= filippo.io/age v1.2.1 h1:X0TZjehAZylOIj4DubWYU1vWQxv9bJpo+Uu2/LGhi1o= filippo.io/age v1.2.1/go.mod h1:JL9ew2lTN+Pyft4RiNGguFfOpewKwSHm5ayKD/A4004= -github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA= -github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= +github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ= +github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= github.com/cloudflare/circl v1.6.0 h1:cr5JKic4HI+LkINy2lg3W2jF8sHCVTBncJr5gIIq7qk= github.com/cloudflare/circl v1.6.0/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZh3pJrofs= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= diff --git a/vendor/github.com/andybalholm/brotli/.gitignore b/vendor/github.com/andybalholm/brotli/.gitignore new file mode 100644 index 0000000..7fd1752 --- /dev/null +++ b/vendor/github.com/andybalholm/brotli/.gitignore @@ -0,0 +1,2 @@ +cpu.out +brotli.test diff --git a/vendor/github.com/andybalholm/brotli/compress_fragment_two_pass.go b/vendor/github.com/andybalholm/brotli/compress_fragment_two_pass.go index 79f9c7f..c5c663a 100644 --- a/vendor/github.com/andybalholm/brotli/compress_fragment_two_pass.go +++ b/vendor/github.com/andybalholm/brotli/compress_fragment_two_pass.go @@ -229,6 +229,8 @@ func storeMetaBlockHeaderBW(len uint, is_uncompressed bool, bw *bitWriter) { nibbles = 4 } else if len <= 1<<20 { nibbles = 5 + } else if len > 1<<24 { + panic("metablock too long") } bw.writeBits(2, uint64(nibbles)-4) diff --git a/vendor/github.com/andybalholm/brotli/find_match_length.go b/vendor/github.com/andybalholm/brotli/find_match_length.go index 09d2ae6..b46f88e 100644 --- a/vendor/github.com/andybalholm/brotli/find_match_length.go +++ b/vendor/github.com/andybalholm/brotli/find_match_length.go @@ -17,7 +17,7 @@ func findMatchLengthWithLimit(s1 []byte, s2 []byte, limit uint) uint { var matched uint = 0 _, _ = s1[limit-1], s2[limit-1] // bounds check switch runtime.GOARCH { - case "amd64": + case "amd64", "arm64": // Compare 8 bytes at at time. for matched+8 <= limit { w1 := binary.LittleEndian.Uint64(s1[matched:]) diff --git a/vendor/github.com/andybalholm/brotli/matchfinder/m4.go b/vendor/github.com/andybalholm/brotli/matchfinder/m4.go index 8189472..f556a01 100644 --- a/vendor/github.com/andybalholm/brotli/matchfinder/m4.go +++ b/vendor/github.com/andybalholm/brotli/matchfinder/m4.go @@ -1,6 +1,7 @@ package matchfinder import ( + "bytes" "encoding/binary" "math/bits" "runtime" @@ -42,7 +43,7 @@ type M4 struct { DistanceBitCost int table []uint32 - chain []uint16 + chain []uint32 history []byte } @@ -100,7 +101,7 @@ func (q *M4) FindMatches(dst []Match, src []byte) []Match { e.NextEmit = len(q.history) q.history = append(q.history, src...) if q.ChainLength > 0 { - q.chain = append(q.chain, make([]uint16, len(src))...) + q.chain = append(q.chain, make([]uint32, len(src))...) } src = q.history @@ -123,15 +124,25 @@ func (q *M4) FindMatches(dst []Match, src []byte) []Match { matches = [3]absoluteMatch{} } + // Look for a repeat match one byte after the current position. + if matches[0] == (absoluteMatch{}) && len(e.Dst) > 0 { + prevDistance := e.Dst[len(e.Dst)-1].Distance + if binary.LittleEndian.Uint32(src[i+1:]) == binary.LittleEndian.Uint32(src[i+1-prevDistance:]) { + // We have a 4-byte match. + m := extendMatch2(src, i+1, i+1-prevDistance, e.NextEmit+1) + if m.End-m.Start >= q.MinLength { + matches[0] = m + } + } + } + // Calculate and store the hash. h := ((binary.LittleEndian.Uint64(src[i:]) & (1<<(8*q.HashLen) - 1)) * hashMul64) >> (64 - q.TableBits) candidate := int(q.table[h]) q.table[h] = uint32(i) if q.ChainLength > 0 && candidate != 0 { delta := i - candidate - if delta < 1<<16 { - q.chain[i] = uint16(delta) - } + q.chain[i] = uint32(delta) } if i < matches[0].End && i != matches[0].End+2-q.HashLen { @@ -220,6 +231,25 @@ func (q *M4) FindMatches(dst []Match, src []byte) []Match { // Emit the first match, shortening it if necessary to avoid overlap with the second. if matches[2].End > matches[1].Start { matches[2].End = matches[1].Start + if q.ChainLength > 0 && matches[2].End-matches[2].Start >= q.MinLength { + // Since the match length was trimmed, we may be able to find a closer match + // to replace it. + pos := matches[2].Start + for { + delta := int(q.chain[pos]) + if delta == 0 { + break + } + pos -= delta + if pos <= matches[2].Match { + break + } + if bytes.Equal(src[matches[2].Start:matches[2].End], src[pos:pos+matches[2].End-matches[2].Start]) { + matches[2].Match = pos + break + } + } + } } if matches[2].End-matches[2].Start >= q.MinLength && q.score(matches[2]) > 0 { e.emit(matches[2]) @@ -261,7 +291,7 @@ const hashMul64 = 0x1E35A7BD1E35A7BD // 0 <= i && i < j && j <= len(src) func extendMatch(src []byte, i, j int) int { switch runtime.GOARCH { - case "amd64": + case "amd64", "arm64": // As long as we are 8 or more bytes before the end of src, we can load and // compare 8 bytes at a time. If those 8 bytes are equal, repeat. for j+8 < len(src) { diff --git a/vendor/github.com/andybalholm/brotli/matchfinder/pathfinder.go b/vendor/github.com/andybalholm/brotli/matchfinder/pathfinder.go new file mode 100644 index 0000000..2a12ab9 --- /dev/null +++ b/vendor/github.com/andybalholm/brotli/matchfinder/pathfinder.go @@ -0,0 +1,328 @@ +package matchfinder + +import ( + "encoding/binary" + "math" + "math/bits" + "slices" +) + +// Pathfinder is a MatchFinder that uses hash chains to find matches, and a +// shortest-path optimizer to choose which matches to use. +type Pathfinder struct { + // MaxDistance is the maximum distance (in bytes) to look back for + // a match. The default is 65535. + MaxDistance int + + // MinLength is the length of the shortest match to return. + // The default is 4. + MinLength int + + // HashLen is the number of bytes to use to calculate the hashes. + // The maximum is 8 and the default is 6. + HashLen int + + // TableBits is the number of bits in the hash table indexes. + // The default is 17 (128K entries). + TableBits int + + // ChainLength is how many entries to search on the "match chain" of older + // locations with the same hash as the current location. + ChainLength int + + table []uint32 + chain []uint32 + + history []byte + + // holding onto buffers to reduce allocations: + + arrivals []arrival + foundMatches []absoluteMatch + matches []Match +} + +func (q *Pathfinder) Reset() { + for i := range q.table { + q.table[i] = 0 + } + q.history = q.history[:0] + q.chain = q.chain[:0] +} + +// An arrival represents how we got to a certain byte position. +// The cost is the total cost to get there from the beginning of the block. +// If distance > 0, the arrival is with a match. +// If distance == 0, the arrival is with a run of literals. +type arrival struct { + length uint32 + distance uint32 + cost float32 +} + +const ( + baseMatchCost float32 = 4 +) + +func (q *Pathfinder) FindMatches(dst []Match, src []byte) []Match { + if q.MaxDistance == 0 { + q.MaxDistance = 65535 + } + if q.MinLength == 0 { + q.MinLength = 4 + } + if q.HashLen == 0 { + q.HashLen = 6 + } + if q.TableBits == 0 { + q.TableBits = 17 + } + if len(q.table) < 1< q.MaxDistance*2 { + // Trim down the history buffer. + delta := len(q.history) - q.MaxDistance + copy(q.history, q.history[delta:]) + q.history = q.history[:q.MaxDistance] + q.chain = q.chain[:q.MaxDistance] + + for i, v := range q.table { + newV := max(int(v)-delta, 0) + q.table[i] = uint32(newV) + } + } + + // Append src to the history buffer. + historyLen := len(q.history) + q.history = append(q.history, src...) + q.chain = append(q.chain, make([]uint32, len(src))...) + src = q.history + + // Calculate hashes and build the chain. + for i := historyLen; i < len(src)-7; i++ { + h := ((binary.LittleEndian.Uint64(src[i:]) & (1<<(8*q.HashLen) - 1)) * hashMul64) >> (64 - q.TableBits) + candidate := int(q.table[h]) + q.table[h] = uint32(i) + if candidate != 0 { + delta := i - candidate + q.chain[i] = uint32(delta) + } + } + + // Look for matches, and collect them in foundMatches. Later we'll figure out + // which ones to actually use. + foundMatches := q.foundMatches[:0] + var prevMatch absoluteMatch + i := historyLen + for i < len(src)-7 { + delta := q.chain[i] + if delta == 0 { + i++ + continue + } + candidate := i - int(delta) + if candidate <= 0 || i-candidate > q.MaxDistance { + i++ + continue + } + + var currentMatch absoluteMatch + + if i >= prevMatch.End && prevMatch != (absoluteMatch{}) { + // Look for a repeat match at i+1. + prevDistance := prevMatch.Start - prevMatch.Match + if binary.LittleEndian.Uint32(src[i+1:]) == binary.LittleEndian.Uint32(src[i+1-prevDistance:]) { + m := extendMatch2(src, i+1, i+1-prevDistance, i+1) + if m.End-m.Start > q.MinLength { + currentMatch = m + foundMatches = append(foundMatches, m) + } + } + } + + if binary.LittleEndian.Uint32(src[candidate:]) == binary.LittleEndian.Uint32(src[i:]) { + m := extendMatch2(src, i, candidate, max(historyLen, prevMatch.Start)) + if m.End-m.Start > q.MinLength { + currentMatch = m + foundMatches = append(foundMatches, m) + } + } + + for range q.ChainLength { + delta := q.chain[candidate] + if delta == 0 { + break + } + candidate -= int(delta) + if candidate <= 0 || i-candidate > q.MaxDistance { + break + } + if binary.LittleEndian.Uint32(src[candidate:]) == binary.LittleEndian.Uint32(src[i:]) { + m := extendMatch2(src, i, candidate, max(historyLen, prevMatch.Start)) + if m.End-m.Start > q.MinLength && m.End-m.Start > currentMatch.End-currentMatch.Start { + currentMatch = m + foundMatches = append(foundMatches, m) + } + } + } + + if i < prevMatch.End && currentMatch.End-currentMatch.Start <= prevMatch.End-prevMatch.Start { + // We were looking for an overlapping match, but we didn't find one longer + // than the previous match. So we'll go back to sequential search, + // starting right after the previous match. + i = prevMatch.End + continue + } + + if currentMatch == (absoluteMatch{}) { + // No match found. Continue with sequential search. + i++ + continue + } + + // We've found a match; now look for matches overlapping the end of it. + prevMatch = currentMatch + i = currentMatch.End + 2 - q.HashLen + } + + q.foundMatches = foundMatches + + slices.SortFunc(foundMatches, func(a, b absoluteMatch) int { return a.Start - b.Start }) + matchIndex := 0 + var pending absoluteMatch + + for i := historyLen; i < len(src); i++ { + var arrivedHere arrival + if i > historyLen { + arrivedHere = arrivals[i-historyLen-1] + } + + unmatched := 0 + if arrivedHere.distance == 0 { + unmatched = int(arrivedHere.length) + } + prevDistance := 0 + if i-unmatched > historyLen { + prevDistance = int(arrivals[i-historyLen-1-unmatched].distance) + } + + literalCost := byteCost[src[i]] + nextArrival := &arrivals[i-historyLen] + if nextArrival.cost == 0 || arrivedHere.cost+literalCost < nextArrival.cost { + *nextArrival = arrival{ + cost: arrivedHere.cost + literalCost, + length: uint32(unmatched + 1), + } + } + + for matchIndex < len(foundMatches) && foundMatches[matchIndex].Start == i { + m := foundMatches[matchIndex] + matchIndex++ + if m.End > pending.End { + pending = m + } + matchCost := baseMatchCost + float32(bits.Len(uint(unmatched))) + if m.Start-m.Match != prevDistance { + matchCost += float32(bits.Len(uint(m.Start - m.Match))) + } + for j := m.Start + q.MinLength; j <= m.End; j++ { + adjustedCost := matchCost + if j-m.Start < 6 { + // Matches shorter than 6 are comparatively rare, and therefore + // have longer codes. + adjustedCost += float32(6-(j-m.Start)) * 2 + } + a := &arrivals[j-historyLen-1] + if a.cost == 0 || arrivedHere.cost+adjustedCost < a.cost { + *a = arrival{ + length: uint32(j - m.Start), + distance: uint32(m.Start - m.Match), + cost: arrivedHere.cost + adjustedCost, + } + } + } + } + + // If a match from an earlier position extends far enough past the current + // position, try using the tail of it, starting from here. + if unmatched == 0 && pending.Start != i && pending.End >= i+q.MinLength && + !(arrivedHere.length != 0 && arrivedHere.distance == uint32(pending.Start-pending.Match)) { + matchCost := baseMatchCost + float32(bits.Len(uint(pending.Start-pending.Match))) + for j := i + q.MinLength; j <= pending.End; j++ { + adjustedCost := matchCost + if j-i < 6 { + // Matches shorter than 6 are comparatively rare, and therefore + // have longer codes. + adjustedCost += float32(6-(j-i)) * 2 + } + a := &arrivals[j-historyLen-1] + if a.cost == 0 || arrivedHere.cost+adjustedCost < a.cost { + *a = arrival{ + length: uint32(j - i), + distance: uint32(pending.Start - pending.Match), + cost: arrivedHere.cost + adjustedCost, + } + } + } + } + + delta := q.chain[i] + if delta == 0 { + continue + } + candidate := i - int(delta) + if candidate <= 0 || i-candidate > q.MaxDistance { + continue + } + } + + // We've found the shortest path; now walk it backward and store the matches. + matches := q.matches[:0] + i = len(arrivals) - 1 + for i >= 0 { + a := arrivals[i] + if a.distance > 0 { + matches = append(matches, Match{ + Length: int(a.length), + Distance: int(a.distance), + }) + i -= int(a.length) + } else { + if len(matches) == 0 { + matches = append(matches, Match{}) + } + matches[len(matches)-1].Unmatched = int(a.length) + i -= int(a.length) + } + } + q.matches = matches + + slices.Reverse(matches) + + return append(dst, matches...) +} diff --git a/vendor/github.com/andybalholm/brotli/reader.go b/vendor/github.com/andybalholm/brotli/reader.go index 9419c79..3e22789 100644 --- a/vendor/github.com/andybalholm/brotli/reader.go +++ b/vendor/github.com/andybalholm/brotli/reader.go @@ -49,6 +49,9 @@ func (r *Reader) Read(p []byte) (n int, err error) { if !decoderHasMoreOutput(r) && len(r.in) == 0 { m, readErr := r.src.Read(r.buf) if m == 0 { + if readErr == io.EOF && r.state != stateDone { + readErr = io.ErrUnexpectedEOF + } // If readErr is `nil`, we just proxy underlying stream behavior. return 0, readErr } diff --git a/vendor/github.com/andybalholm/brotli/writer.go b/vendor/github.com/andybalholm/brotli/writer.go index 8a68811..277f252 100644 --- a/vendor/github.com/andybalholm/brotli/writer.go +++ b/vendor/github.com/andybalholm/brotli/writer.go @@ -121,18 +121,18 @@ type nopCloser struct { func (nopCloser) Close() error { return nil } // NewWriterV2 is like NewWriterLevel, but it uses the new implementation -// based on the matchfinder package. It currently supports up to level 7; -// if a higher level is specified, level 7 will be used. +// based on the matchfinder package. It currently supports up to level 9; +// if a higher level is specified, level 9 will be used. func NewWriterV2(dst io.Writer, level int) *matchfinder.Writer { var mf matchfinder.MatchFinder if level < 2 { mf = matchfinder.M0{Lazy: level == 1} - } else { + } else if level < 8 { hashLen := 6 if level >= 6 { hashLen = 5 } - chainLen := 64 + chainLen := 16 switch level { case 2: chainLen = 0 @@ -149,7 +149,19 @@ func NewWriterV2(dst io.Writer, level int) *matchfinder.Writer { MaxDistance: 1 << 20, ChainLength: chainLen, HashLen: hashLen, - DistanceBitCost: 57, + DistanceBitCost: 66, + } + } else { + chainLen := 32 + hashLen := 5 + if level == 8 { + chainLen = 4 + hashLen = 6 + } + mf = &matchfinder.Pathfinder{ + MaxDistance: 1 << 20, + ChainLength: chainLen, + HashLen: hashLen, } } diff --git a/vendor/modules.txt b/vendor/modules.txt index 36e26bd..2f69b7e 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -4,8 +4,8 @@ filippo.io/age filippo.io/age/internal/bech32 filippo.io/age/internal/format filippo.io/age/internal/stream -# github.com/andybalholm/brotli v1.1.1 -## explicit; go 1.13 +# github.com/andybalholm/brotli v1.2.0 +## explicit; go 1.22 github.com/andybalholm/brotli github.com/andybalholm/brotli/matchfinder # github.com/cloudflare/circl v1.6.0