mirror of
https://gitee.com/johng/gf
synced 2026-06-07 10:22:11 +08:00
新增go modules支持,自行管理第三方包依赖,方便开发者使用
This commit is contained in:
33
third/github.com/pierrec/lz4/.gitignore
vendored
Normal file
33
third/github.com/pierrec/lz4/.gitignore
vendored
Normal file
@ -0,0 +1,33 @@
|
||||
# Created by https://www.gitignore.io/api/macos
|
||||
|
||||
### macOS ###
|
||||
*.DS_Store
|
||||
.AppleDouble
|
||||
.LSOverride
|
||||
|
||||
# Icon must end with two \r
|
||||
Icon
|
||||
|
||||
|
||||
# Thumbnails
|
||||
._*
|
||||
|
||||
# Files that might appear in the root of a volume
|
||||
.DocumentRevisions-V100
|
||||
.fseventsd
|
||||
.Spotlight-V100
|
||||
.TemporaryItems
|
||||
.Trashes
|
||||
.VolumeIcon.icns
|
||||
.com.apple.timemachine.donotpresent
|
||||
|
||||
# Directories potentially created on remote AFP share
|
||||
.AppleDB
|
||||
.AppleDesktop
|
||||
Network Trash Folder
|
||||
Temporary Items
|
||||
.apdisk
|
||||
|
||||
# End of https://www.gitignore.io/api/macos
|
||||
|
||||
lz4c/lz4c
|
||||
18
third/github.com/pierrec/lz4/.travis.yml
Normal file
18
third/github.com/pierrec/lz4/.travis.yml
Normal file
@ -0,0 +1,18 @@
|
||||
language: go
|
||||
|
||||
go:
|
||||
- 1.8.x
|
||||
- 1.9.x
|
||||
- 1.10.x
|
||||
- master
|
||||
|
||||
matrix:
|
||||
fast_finish: true
|
||||
allow_failures:
|
||||
- go: master
|
||||
|
||||
sudo: false
|
||||
|
||||
script:
|
||||
- go test -v -cpu=2
|
||||
- go test -v -cpu=2 -race
|
||||
28
third/github.com/pierrec/lz4/LICENSE
Normal file
28
third/github.com/pierrec/lz4/LICENSE
Normal file
@ -0,0 +1,28 @@
|
||||
Copyright (c) 2015, Pierre Curto
|
||||
All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright notice, this
|
||||
list of conditions and the following disclaimer.
|
||||
|
||||
* Redistributions in binary form must reproduce the above copyright notice,
|
||||
this list of conditions and the following disclaimer in the documentation
|
||||
and/or other materials provided with the distribution.
|
||||
|
||||
* Neither the name of xxHash nor the names of its
|
||||
contributors may be used to endorse or promote products derived from
|
||||
this software without specific prior written permission.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
22
third/github.com/pierrec/lz4/README.md
Normal file
22
third/github.com/pierrec/lz4/README.md
Normal file
@ -0,0 +1,22 @@
|
||||
[](https://godoc.org/github.com/pierrec/lz4)
|
||||
|
||||
# lz4
|
||||
LZ4 compression and decompression in pure Go.
|
||||
|
||||
## Usage
|
||||
|
||||
```go
|
||||
import "gitee.com/johng/gf/third/github.com/pierrec/lz4"
|
||||
```
|
||||
|
||||
## Description
|
||||
Package lz4 implements reading and writing lz4 compressed data (a frame),
|
||||
as specified in http://fastcompression.blogspot.fr/2013/04/lz4-streaming-format-final.html.
|
||||
Although the block level compression and decompression functions are exposed and are fully compatible
|
||||
with the lz4 block format definition, they are low level and should not be used directly.
|
||||
|
||||
For a complete description of an lz4 compressed block, see:
|
||||
http://fastcompression.blogspot.fr/2011/05/lz4-explained.html
|
||||
|
||||
See https://github.com/Cyan4973/lz4 for the reference C implementation.
|
||||
|
||||
119
third/github.com/pierrec/lz4/bench_test.go
Normal file
119
third/github.com/pierrec/lz4/bench_test.go
Normal file
@ -0,0 +1,119 @@
|
||||
package lz4_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"testing"
|
||||
|
||||
"gitee.com/johng/gf/third/github.com/pierrec/lz4"
|
||||
)
|
||||
|
||||
func BenchmarkCompress(b *testing.B) {
|
||||
var hashTable [1 << 16]int
|
||||
buf := make([]byte, len(pg1661))
|
||||
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
|
||||
for i := 0; i < b.N; i++ {
|
||||
lz4.CompressBlock(pg1661, buf, hashTable[:])
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkCompressHC(b *testing.B) {
|
||||
buf := make([]byte, len(pg1661))
|
||||
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
|
||||
for i := 0; i < b.N; i++ {
|
||||
lz4.CompressBlockHC(pg1661, buf, 16)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkUncompress(b *testing.B) {
|
||||
buf := make([]byte, len(pg1661))
|
||||
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
|
||||
for i := 0; i < b.N; i++ {
|
||||
lz4.UncompressBlock(pg1661LZ4, buf)
|
||||
}
|
||||
}
|
||||
|
||||
func mustLoadFile(f string) []byte {
|
||||
b, err := ioutil.ReadFile(f)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
var (
|
||||
pg1661 = mustLoadFile("testdata/pg1661.txt")
|
||||
digits = mustLoadFile("testdata/e.txt")
|
||||
twain = mustLoadFile("testdata/Mark.Twain-Tom.Sawyer.txt")
|
||||
random = mustLoadFile("testdata/random.data")
|
||||
pg1661LZ4 = mustLoadFile("testdata/pg1661.txt.lz4")
|
||||
digitsLZ4 = mustLoadFile("testdata/e.txt.lz4")
|
||||
twainLZ4 = mustLoadFile("testdata/Mark.Twain-Tom.Sawyer.txt.lz4")
|
||||
randomLZ4 = mustLoadFile("testdata/random.data.lz4")
|
||||
)
|
||||
|
||||
func benchmarkUncompress(b *testing.B, compressed []byte) {
|
||||
r := bytes.NewReader(compressed)
|
||||
zr := lz4.NewReader(r)
|
||||
|
||||
// Determine the uncompressed size of testfile.
|
||||
uncompressedSize, err := io.Copy(ioutil.Discard, zr)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
|
||||
b.SetBytes(uncompressedSize)
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
|
||||
for i := 0; i < b.N; i++ {
|
||||
r.Reset(compressed)
|
||||
zr.Reset(r)
|
||||
io.Copy(ioutil.Discard, zr)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkUncompressPg1661(b *testing.B) { benchmarkUncompress(b, pg1661LZ4) }
|
||||
func BenchmarkUncompressDigits(b *testing.B) { benchmarkUncompress(b, digitsLZ4) }
|
||||
func BenchmarkUncompressTwain(b *testing.B) { benchmarkUncompress(b, twainLZ4) }
|
||||
func BenchmarkUncompressRand(b *testing.B) { benchmarkUncompress(b, randomLZ4) }
|
||||
|
||||
func benchmarkCompress(b *testing.B, uncompressed []byte) {
|
||||
w := bytes.NewBuffer(nil)
|
||||
zw := lz4.NewWriter(w)
|
||||
r := bytes.NewReader(uncompressed)
|
||||
|
||||
// Determine the compressed size of testfile.
|
||||
compressedSize, err := io.Copy(zw, r)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
if err := zw.Close(); err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
|
||||
b.SetBytes(compressedSize)
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
|
||||
for i := 0; i < b.N; i++ {
|
||||
r.Reset(uncompressed)
|
||||
zw.Reset(w)
|
||||
io.Copy(zw, r)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkCompressPg1661(b *testing.B) { benchmarkCompress(b, pg1661) }
|
||||
func BenchmarkCompressDigits(b *testing.B) { benchmarkCompress(b, digits) }
|
||||
func BenchmarkCompressTwain(b *testing.B) { benchmarkCompress(b, twain) }
|
||||
func BenchmarkCompressRand(b *testing.B) { benchmarkCompress(b, random) }
|
||||
397
third/github.com/pierrec/lz4/block.go
Normal file
397
third/github.com/pierrec/lz4/block.go
Normal file
@ -0,0 +1,397 @@
|
||||
package lz4
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
)
|
||||
|
||||
var (
|
||||
// ErrInvalidSourceShortBuffer is returned by UncompressBlock or CompressBLock when a compressed
|
||||
// block is corrupted or the destination buffer is not large enough for the uncompressed data.
|
||||
ErrInvalidSourceShortBuffer = errors.New("lz4: invalid source or destination buffer too short")
|
||||
// ErrInvalid is returned when reading an invalid LZ4 archive.
|
||||
ErrInvalid = errors.New("lz4: bad magic number")
|
||||
)
|
||||
|
||||
// blockHash hashes 4 bytes into a value < winSize.
|
||||
func blockHash(x uint32) uint32 {
|
||||
const hasher uint32 = 2654435761 // Knuth multiplicative hash.
|
||||
return x * hasher >> hashShift
|
||||
}
|
||||
|
||||
// CompressBlockBound returns the maximum size of a given buffer of size n, when not compressible.
|
||||
func CompressBlockBound(n int) int {
|
||||
return n + n/255 + 16
|
||||
}
|
||||
|
||||
// UncompressBlock uncompresses the source buffer into the destination one,
|
||||
// and returns the uncompressed size.
|
||||
//
|
||||
// The destination buffer must be sized appropriately.
|
||||
//
|
||||
// An error is returned if the source data is invalid or the destination buffer is too small.
|
||||
func UncompressBlock(src, dst []byte) (si int, err error) {
|
||||
defer func() {
|
||||
// It is now faster to let the runtime panic and recover on out of bound slice access
|
||||
// than checking indices as we go along.
|
||||
if recover() != nil {
|
||||
err = ErrInvalidSourceShortBuffer
|
||||
}
|
||||
}()
|
||||
sn := len(src)
|
||||
if sn == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
var di int
|
||||
|
||||
for {
|
||||
// Literals and match lengths (token).
|
||||
b := int(src[si])
|
||||
si++
|
||||
|
||||
// Literals.
|
||||
if lLen := b >> 4; lLen > 0 {
|
||||
if lLen == 0xF {
|
||||
for src[si] == 0xFF {
|
||||
lLen += 0xFF
|
||||
si++
|
||||
}
|
||||
lLen += int(src[si])
|
||||
si++
|
||||
}
|
||||
i := si
|
||||
si += lLen
|
||||
di += copy(dst[di:], src[i:si])
|
||||
|
||||
if si >= sn {
|
||||
return di, nil
|
||||
}
|
||||
}
|
||||
|
||||
si++
|
||||
_ = src[si] // Bound check elimination.
|
||||
offset := int(src[si-1]) | int(src[si])<<8
|
||||
si++
|
||||
|
||||
// Match.
|
||||
mLen := b & 0xF
|
||||
if mLen == 0xF {
|
||||
for src[si] == 0xFF {
|
||||
mLen += 0xFF
|
||||
si++
|
||||
}
|
||||
mLen += int(src[si])
|
||||
si++
|
||||
}
|
||||
mLen += minMatch
|
||||
|
||||
// Copy the match.
|
||||
i := di - offset
|
||||
if offset > 0 && mLen >= offset {
|
||||
// Efficiently copy the match dst[di-offset:di] into the dst slice.
|
||||
bytesToCopy := offset * (mLen / offset)
|
||||
expanded := dst[i:]
|
||||
for n := offset; n <= bytesToCopy+offset; n *= 2 {
|
||||
copy(expanded[n:], expanded[:n])
|
||||
}
|
||||
di += bytesToCopy
|
||||
mLen -= bytesToCopy
|
||||
}
|
||||
di += copy(dst[di:], dst[i:i+mLen])
|
||||
}
|
||||
}
|
||||
|
||||
// CompressBlock compresses the source buffer into the destination one.
|
||||
// This is the fast version of LZ4 compression and also the default one.
|
||||
// The size of hashTable must be at least 64Kb.
|
||||
//
|
||||
// The size of the compressed data is returned. If it is 0 and no error, then the data is incompressible.
|
||||
//
|
||||
// An error is returned if the destination buffer is too small.
|
||||
func CompressBlock(src, dst []byte, hashTable []int) (di int, err error) {
|
||||
defer func() {
|
||||
if recover() != nil {
|
||||
err = ErrInvalidSourceShortBuffer
|
||||
}
|
||||
}()
|
||||
|
||||
sn, dn := len(src)-mfLimit, len(dst)
|
||||
if sn <= 0 || dn == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
var si int
|
||||
|
||||
// Fast scan strategy: the hash table only stores the last 4 bytes sequences.
|
||||
// const accInit = 1 << skipStrength
|
||||
|
||||
anchor := si // Position of the current literals.
|
||||
// acc := accInit // Variable step: improves performance on non-compressible data.
|
||||
|
||||
for si < sn {
|
||||
// Hash the next 4 bytes (sequence)...
|
||||
match := binary.LittleEndian.Uint32(src[si:])
|
||||
h := blockHash(match)
|
||||
|
||||
ref := hashTable[h]
|
||||
hashTable[h] = si
|
||||
if ref >= sn { // Invalid reference (dirty hashtable).
|
||||
si++
|
||||
continue
|
||||
}
|
||||
offset := si - ref
|
||||
if offset <= 0 || offset >= winSize || // Out of window.
|
||||
match != binary.LittleEndian.Uint32(src[ref:]) { // Hash collision on different matches.
|
||||
// si += acc >> skipStrength
|
||||
// acc++
|
||||
si++
|
||||
continue
|
||||
}
|
||||
|
||||
// Match found.
|
||||
// acc = accInit
|
||||
lLen := si - anchor // Literal length.
|
||||
|
||||
// Encode match length part 1.
|
||||
si += minMatch
|
||||
mLen := si // Match length has minMatch already.
|
||||
// Find the longest match, first looking by batches of 8 bytes.
|
||||
for si < sn && binary.LittleEndian.Uint64(src[si:]) == binary.LittleEndian.Uint64(src[si-offset:]) {
|
||||
si += 8
|
||||
}
|
||||
// Then byte by byte.
|
||||
for si < sn && src[si] == src[si-offset] {
|
||||
si++
|
||||
}
|
||||
|
||||
mLen = si - mLen
|
||||
if mLen < 0xF {
|
||||
dst[di] = byte(mLen)
|
||||
} else {
|
||||
dst[di] = 0xF
|
||||
}
|
||||
|
||||
// Encode literals length.
|
||||
if lLen < 0xF {
|
||||
dst[di] |= byte(lLen << 4)
|
||||
} else {
|
||||
dst[di] |= 0xF0
|
||||
di++
|
||||
l := lLen - 0xF
|
||||
for ; l >= 0xFF; l -= 0xFF {
|
||||
dst[di] = 0xFF
|
||||
di++
|
||||
}
|
||||
dst[di] = byte(l)
|
||||
}
|
||||
di++
|
||||
|
||||
// Literals.
|
||||
copy(dst[di:], src[anchor:anchor+lLen])
|
||||
di += lLen + 2
|
||||
anchor = si
|
||||
|
||||
// Encode offset.
|
||||
_ = dst[di] // Bound check elimination.
|
||||
dst[di-2], dst[di-1] = byte(offset), byte(offset>>8)
|
||||
|
||||
// Encode match length part 2.
|
||||
if mLen >= 0xF {
|
||||
for mLen -= 0xF; mLen >= 0xFF; mLen -= 0xFF {
|
||||
dst[di] = 0xFF
|
||||
di++
|
||||
}
|
||||
dst[di] = byte(mLen)
|
||||
di++
|
||||
}
|
||||
}
|
||||
|
||||
if anchor == 0 {
|
||||
// Incompressible.
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
// Last literals.
|
||||
lLen := len(src) - anchor
|
||||
if lLen < 0xF {
|
||||
dst[di] = byte(lLen << 4)
|
||||
} else {
|
||||
dst[di] = 0xF0
|
||||
di++
|
||||
for lLen -= 0xF; lLen >= 0xFF; lLen -= 0xFF {
|
||||
dst[di] = 0xFF
|
||||
di++
|
||||
}
|
||||
dst[di] = byte(lLen)
|
||||
}
|
||||
di++
|
||||
|
||||
// Write the last literals.
|
||||
if di >= anchor {
|
||||
// Incompressible.
|
||||
return 0, nil
|
||||
}
|
||||
di += copy(dst[di:], src[anchor:])
|
||||
return di, nil
|
||||
}
|
||||
|
||||
// CompressBlockHC compresses the source buffer src into the destination dst
|
||||
// with max search depth (use 0 or negative value for no max).
|
||||
//
|
||||
// CompressBlockHC compression ratio is better than CompressBlock but it is also slower.
|
||||
//
|
||||
// The size of the compressed data is returned. If it is 0 and no error, then the data is not compressible.
|
||||
//
|
||||
// An error is returned if the destination buffer is too small.
|
||||
func CompressBlockHC(src, dst []byte, depth int) (di int, err error) {
|
||||
defer func() {
|
||||
if recover() != nil {
|
||||
err = ErrInvalidSourceShortBuffer
|
||||
}
|
||||
}()
|
||||
|
||||
sn, dn := len(src)-mfLimit, len(dst)
|
||||
if sn <= 0 || dn == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
var si int
|
||||
|
||||
// hashTable: stores the last position found for a given hash
|
||||
// chaingTable: stores previous positions for a given hash
|
||||
var hashTable, chainTable [winSize]int
|
||||
|
||||
if depth <= 0 {
|
||||
depth = winSize
|
||||
}
|
||||
|
||||
anchor := si
|
||||
for si < sn {
|
||||
// Hash the next 4 bytes (sequence).
|
||||
match := binary.LittleEndian.Uint32(src[si:])
|
||||
h := blockHash(match)
|
||||
|
||||
// Follow the chain until out of window and give the longest match.
|
||||
mLen := 0
|
||||
offset := 0
|
||||
for next, try := hashTable[h], depth; try > 0 && next > 0 && si-next < winSize; next = chainTable[next&winMask] {
|
||||
// The first (mLen==0) or next byte (mLen>=minMatch) at current match length
|
||||
// must match to improve on the match length.
|
||||
if src[next+mLen] != src[si+mLen] {
|
||||
continue
|
||||
}
|
||||
ml := 0
|
||||
// Compare the current position with a previous with the same hash.
|
||||
for ml < sn-si && binary.LittleEndian.Uint64(src[next+ml:]) == binary.LittleEndian.Uint64(src[si+ml:]) {
|
||||
ml += 8
|
||||
}
|
||||
for ml < sn-si && src[next+ml] == src[si+ml] {
|
||||
ml++
|
||||
}
|
||||
if ml+1 < minMatch || ml <= mLen {
|
||||
// Match too small (<minMath) or smaller than the current match.
|
||||
continue
|
||||
}
|
||||
// Found a longer match, keep its position and length.
|
||||
mLen = ml
|
||||
offset = si - next
|
||||
// Try another previous position with the same hash.
|
||||
try--
|
||||
}
|
||||
chainTable[si&winMask] = hashTable[h]
|
||||
hashTable[h] = si
|
||||
|
||||
// No match found.
|
||||
if mLen == 0 {
|
||||
si++
|
||||
continue
|
||||
}
|
||||
|
||||
// Match found.
|
||||
// Update hash/chain tables with overlapping bytes:
|
||||
// si already hashed, add everything from si+1 up to the match length.
|
||||
winStart := si + 1
|
||||
if ws := si + mLen - winSize; ws > winStart {
|
||||
winStart = ws
|
||||
}
|
||||
for si, ml := winStart, si+mLen; si < ml; {
|
||||
match >>= 8
|
||||
match |= uint32(src[si+3]) << 24
|
||||
h := blockHash(match)
|
||||
chainTable[si&winMask] = hashTable[h]
|
||||
hashTable[h] = si
|
||||
si++
|
||||
}
|
||||
|
||||
lLen := si - anchor
|
||||
si += mLen
|
||||
mLen -= minMatch // Match length does not include minMatch.
|
||||
|
||||
if mLen < 0xF {
|
||||
dst[di] = byte(mLen)
|
||||
} else {
|
||||
dst[di] = 0xF
|
||||
}
|
||||
|
||||
// Encode literals length.
|
||||
if lLen < 0xF {
|
||||
dst[di] |= byte(lLen << 4)
|
||||
} else {
|
||||
dst[di] |= 0xF0
|
||||
di++
|
||||
l := lLen - 0xF
|
||||
for ; l >= 0xFF; l -= 0xFF {
|
||||
dst[di] = 0xFF
|
||||
di++
|
||||
}
|
||||
dst[di] = byte(l)
|
||||
}
|
||||
di++
|
||||
|
||||
// Literals.
|
||||
copy(dst[di:], src[anchor:anchor+lLen])
|
||||
di += lLen
|
||||
anchor = si
|
||||
|
||||
// Encode offset.
|
||||
di += 2
|
||||
dst[di-2], dst[di-1] = byte(offset), byte(offset>>8)
|
||||
|
||||
// Encode match length part 2.
|
||||
if mLen >= 0xF {
|
||||
for mLen -= 0xF; mLen >= 0xFF; mLen -= 0xFF {
|
||||
dst[di] = 0xFF
|
||||
di++
|
||||
}
|
||||
dst[di] = byte(mLen)
|
||||
di++
|
||||
}
|
||||
}
|
||||
|
||||
if anchor == 0 {
|
||||
// Incompressible.
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
// Last literals.
|
||||
lLen := len(src) - anchor
|
||||
if lLen < 0xF {
|
||||
dst[di] = byte(lLen << 4)
|
||||
} else {
|
||||
dst[di] = 0xF0
|
||||
di++
|
||||
lLen -= 0xF
|
||||
for ; lLen >= 0xFF; lLen -= 0xFF {
|
||||
dst[di] = 0xFF
|
||||
di++
|
||||
}
|
||||
dst[di] = byte(lLen)
|
||||
}
|
||||
di++
|
||||
|
||||
// Write the last literals.
|
||||
if di >= anchor {
|
||||
// Incompressible.
|
||||
return 0, nil
|
||||
}
|
||||
di += copy(dst[di:], src[anchor:])
|
||||
return di, nil
|
||||
}
|
||||
98
third/github.com/pierrec/lz4/block_test.go
Normal file
98
third/github.com/pierrec/lz4/block_test.go
Normal file
@ -0,0 +1,98 @@
|
||||
//+build go1.9
|
||||
|
||||
package lz4_test
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"gitee.com/johng/gf/third/github.com/pierrec/lz4"
|
||||
)
|
||||
|
||||
type testcase struct {
|
||||
file string
|
||||
compressible bool
|
||||
src []byte
|
||||
}
|
||||
|
||||
var rawFiles = []testcase{
|
||||
// {"testdata/207326ba-36f8-11e7-954a-aca46ba8ca73.png", true, nil},
|
||||
{"testdata/e.txt", true, nil},
|
||||
{"testdata/gettysburg.txt", true, nil},
|
||||
{"testdata/Mark.Twain-Tom.Sawyer.txt", true, nil},
|
||||
{"testdata/pg1661.txt", true, nil},
|
||||
{"testdata/pi.txt", true, nil},
|
||||
{"testdata/random.data", false, nil},
|
||||
{"testdata/repeat.txt", true, nil},
|
||||
}
|
||||
|
||||
func TestCompressUncompressBlock(t *testing.T) {
|
||||
type compressor func(s, d []byte) (int, error)
|
||||
|
||||
run := func(tc testcase, compress compressor) int {
|
||||
t.Helper()
|
||||
src := tc.src
|
||||
|
||||
// Compress the data.
|
||||
zbuf := make([]byte, lz4.CompressBlockBound(len(src)))
|
||||
n, err := compress(src, zbuf)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
return 0
|
||||
}
|
||||
zbuf = zbuf[:n]
|
||||
|
||||
// Make sure that it was actually compressed unless not compressible.
|
||||
if !tc.compressible {
|
||||
return 0
|
||||
}
|
||||
|
||||
if n == 0 || n >= len(src) {
|
||||
t.Errorf("data not compressed: %d/%d", n, len(src))
|
||||
return 0
|
||||
}
|
||||
|
||||
// Uncompress the data.
|
||||
buf := make([]byte, len(src))
|
||||
n, err = lz4.UncompressBlock(zbuf, buf)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
buf = buf[:n]
|
||||
if !reflect.DeepEqual(src, buf) {
|
||||
t.Error("uncompressed compressed data not matching initial input")
|
||||
return 0
|
||||
}
|
||||
|
||||
return len(zbuf)
|
||||
}
|
||||
|
||||
for _, tc := range rawFiles {
|
||||
src, err := ioutil.ReadFile(tc.file)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tc.src = src
|
||||
|
||||
var n, nhc int
|
||||
t.Run("", func(t *testing.T) {
|
||||
tc := tc
|
||||
t.Run(tc.file, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
n = run(tc, func(src, dst []byte) (int, error) {
|
||||
var ht [1 << 16]int
|
||||
return lz4.CompressBlock(src, dst, ht[:])
|
||||
})
|
||||
})
|
||||
t.Run(fmt.Sprintf("%s HC", tc.file), func(t *testing.T) {
|
||||
t.Parallel()
|
||||
nhc = run(tc, func(src, dst []byte) (int, error) {
|
||||
return lz4.CompressBlockHC(src, dst, -1)
|
||||
})
|
||||
})
|
||||
})
|
||||
fmt.Printf("%-40s: %8d / %8d / %8d\n", tc.file, n, nhc, len(src))
|
||||
}
|
||||
}
|
||||
21
third/github.com/pierrec/lz4/debug.go
Normal file
21
third/github.com/pierrec/lz4/debug.go
Normal file
@ -0,0 +1,21 @@
|
||||
// +build lz4debug
|
||||
|
||||
package lz4
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
)
|
||||
|
||||
func debug(args ...interface{}) {
|
||||
_, file, line, _ := runtime.Caller(1)
|
||||
file = filepath.Base(file)
|
||||
|
||||
f := fmt.Sprintf("LZ4: %s:%d %s", file, line, args[0])
|
||||
if f[len(f)-1] != '\n' {
|
||||
f += "\n"
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, f, args[1:]...)
|
||||
}
|
||||
5
third/github.com/pierrec/lz4/debug_stub.go
Normal file
5
third/github.com/pierrec/lz4/debug_stub.go
Normal file
@ -0,0 +1,5 @@
|
||||
// +build !lz4debug
|
||||
|
||||
package lz4
|
||||
|
||||
func debug(args ...interface{}) {}
|
||||
13
third/github.com/pierrec/lz4/export_test.go
Normal file
13
third/github.com/pierrec/lz4/export_test.go
Normal file
@ -0,0 +1,13 @@
|
||||
// Expose some internals for testing purposes
|
||||
package lz4
|
||||
|
||||
// expose the possible block max sizes
|
||||
var BlockMaxSizeItems []int
|
||||
|
||||
func init() {
|
||||
for s := range bsMapValue {
|
||||
BlockMaxSizeItems = append(BlockMaxSizeItems, s)
|
||||
}
|
||||
}
|
||||
|
||||
var FrameSkipMagic = frameSkipMagic
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@ -0,0 +1 @@
|
||||
"M@5
|
||||
Binary file not shown.
BIN
third/github.com/pierrec/lz4/fuzz/corpus/1.bz2
Executable file
BIN
third/github.com/pierrec/lz4/fuzz/corpus/1.bz2
Executable file
Binary file not shown.
BIN
third/github.com/pierrec/lz4/fuzz/corpus/10.bz2
Executable file
BIN
third/github.com/pierrec/lz4/fuzz/corpus/10.bz2
Executable file
Binary file not shown.
Binary file not shown.
Binary file not shown.
BIN
third/github.com/pierrec/lz4/fuzz/corpus/11.bz2
Executable file
BIN
third/github.com/pierrec/lz4/fuzz/corpus/11.bz2
Executable file
Binary file not shown.
Binary file not shown.
BIN
third/github.com/pierrec/lz4/fuzz/corpus/12.bz2
Executable file
BIN
third/github.com/pierrec/lz4/fuzz/corpus/12.bz2
Executable file
Binary file not shown.
Binary file not shown.
BIN
third/github.com/pierrec/lz4/fuzz/corpus/13.bz2
Executable file
BIN
third/github.com/pierrec/lz4/fuzz/corpus/13.bz2
Executable file
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
BIN
third/github.com/pierrec/lz4/fuzz/corpus/14.bz2
Executable file
BIN
third/github.com/pierrec/lz4/fuzz/corpus/14.bz2
Executable file
Binary file not shown.
Binary file not shown.
Binary file not shown.
BIN
third/github.com/pierrec/lz4/fuzz/corpus/15.bz2
Executable file
BIN
third/github.com/pierrec/lz4/fuzz/corpus/15.bz2
Executable file
Binary file not shown.
Binary file not shown.
Binary file not shown.
BIN
third/github.com/pierrec/lz4/fuzz/corpus/16.bz2
Executable file
BIN
third/github.com/pierrec/lz4/fuzz/corpus/16.bz2
Executable file
Binary file not shown.
BIN
third/github.com/pierrec/lz4/fuzz/corpus/17.bz2
Executable file
BIN
third/github.com/pierrec/lz4/fuzz/corpus/17.bz2
Executable file
Binary file not shown.
Binary file not shown.
Binary file not shown.
BIN
third/github.com/pierrec/lz4/fuzz/corpus/18.bz2
Executable file
BIN
third/github.com/pierrec/lz4/fuzz/corpus/18.bz2
Executable file
Binary file not shown.
Binary file not shown.
BIN
third/github.com/pierrec/lz4/fuzz/corpus/19.bz2
Executable file
BIN
third/github.com/pierrec/lz4/fuzz/corpus/19.bz2
Executable file
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@ -0,0 +1 @@
|
||||
"M
|
||||
Binary file not shown.
@ -0,0 +1 @@
|
||||
"M3
|
||||
Binary file not shown.
Binary file not shown.
BIN
third/github.com/pierrec/lz4/fuzz/corpus/2.bz2
Executable file
BIN
third/github.com/pierrec/lz4/fuzz/corpus/2.bz2
Executable file
Binary file not shown.
BIN
third/github.com/pierrec/lz4/fuzz/corpus/20.bz2
Executable file
BIN
third/github.com/pierrec/lz4/fuzz/corpus/20.bz2
Executable file
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
BIN
third/github.com/pierrec/lz4/fuzz/corpus/21.bz2
Executable file
BIN
third/github.com/pierrec/lz4/fuzz/corpus/21.bz2
Executable file
Binary file not shown.
Binary file not shown.
BIN
third/github.com/pierrec/lz4/fuzz/corpus/22.bz2
Executable file
BIN
third/github.com/pierrec/lz4/fuzz/corpus/22.bz2
Executable file
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
BIN
third/github.com/pierrec/lz4/fuzz/corpus/23.bz2
Executable file
BIN
third/github.com/pierrec/lz4/fuzz/corpus/23.bz2
Executable file
Binary file not shown.
Binary file not shown.
BIN
third/github.com/pierrec/lz4/fuzz/corpus/24.bz2
Executable file
BIN
third/github.com/pierrec/lz4/fuzz/corpus/24.bz2
Executable file
Binary file not shown.
Binary file not shown.
BIN
third/github.com/pierrec/lz4/fuzz/corpus/25.bz2
Executable file
BIN
third/github.com/pierrec/lz4/fuzz/corpus/25.bz2
Executable file
Binary file not shown.
Binary file not shown.
BIN
third/github.com/pierrec/lz4/fuzz/corpus/26.bz2
Executable file
BIN
third/github.com/pierrec/lz4/fuzz/corpus/26.bz2
Executable file
Binary file not shown.
Binary file not shown.
BIN
third/github.com/pierrec/lz4/fuzz/corpus/27.bz2
Executable file
BIN
third/github.com/pierrec/lz4/fuzz/corpus/27.bz2
Executable file
Binary file not shown.
Binary file not shown.
Binary file not shown.
BIN
third/github.com/pierrec/lz4/fuzz/corpus/28.bz2
Executable file
BIN
third/github.com/pierrec/lz4/fuzz/corpus/28.bz2
Executable file
Binary file not shown.
BIN
third/github.com/pierrec/lz4/fuzz/corpus/29.bz2
Executable file
BIN
third/github.com/pierrec/lz4/fuzz/corpus/29.bz2
Executable file
Binary file not shown.
@ -0,0 +1 @@
|
||||
"M1A
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
@ -0,0 +1 @@
|
||||
"MM@"<22><><EFBFBD><EFBFBD>p+[<5B>
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user