新增go modules支持,自行管理第三方包依赖,方便开发者使用

This commit is contained in:
john
2018-10-22 11:13:00 +08:00
parent 77e13a9a16
commit 7fbbf09b0e
1605 changed files with 383807 additions and 25 deletions

33
third/github.com/pierrec/lz4/.gitignore vendored Normal file
View File

@ -0,0 +1,33 @@
# Created by https://www.gitignore.io/api/macos
### macOS ###
*.DS_Store
.AppleDouble
.LSOverride
# Icon must end with two \r
Icon
# Thumbnails
._*
# Files that might appear in the root of a volume
.DocumentRevisions-V100
.fseventsd
.Spotlight-V100
.TemporaryItems
.Trashes
.VolumeIcon.icns
.com.apple.timemachine.donotpresent
# Directories potentially created on remote AFP share
.AppleDB
.AppleDesktop
Network Trash Folder
Temporary Items
.apdisk
# End of https://www.gitignore.io/api/macos
lz4c/lz4c

View File

@ -0,0 +1,18 @@
language: go
go:
- 1.8.x
- 1.9.x
- 1.10.x
- master
matrix:
fast_finish: true
allow_failures:
- go: master
sudo: false
script:
- go test -v -cpu=2
- go test -v -cpu=2 -race

View File

@ -0,0 +1,28 @@
Copyright (c) 2015, Pierre Curto
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
* Neither the name of xxHash nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

View File

@ -0,0 +1,22 @@
[![godoc](https://godoc.org/github.com/pierrec/lz4?status.png)](https://godoc.org/github.com/pierrec/lz4)
# lz4
LZ4 compression and decompression in pure Go.
## Usage
```go
import "gitee.com/johng/gf/third/github.com/pierrec/lz4"
```
## Description
Package lz4 implements reading and writing lz4 compressed data (a frame),
as specified in http://fastcompression.blogspot.fr/2013/04/lz4-streaming-format-final.html.
Although the block level compression and decompression functions are exposed and are fully compatible
with the lz4 block format definition, they are low level and should not be used directly.
For a complete description of an lz4 compressed block, see:
http://fastcompression.blogspot.fr/2011/05/lz4-explained.html
See https://github.com/Cyan4973/lz4 for the reference C implementation.

View File

@ -0,0 +1,119 @@
package lz4_test
import (
"bytes"
"io"
"io/ioutil"
"testing"
"gitee.com/johng/gf/third/github.com/pierrec/lz4"
)
func BenchmarkCompress(b *testing.B) {
var hashTable [1 << 16]int
buf := make([]byte, len(pg1661))
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
lz4.CompressBlock(pg1661, buf, hashTable[:])
}
}
func BenchmarkCompressHC(b *testing.B) {
buf := make([]byte, len(pg1661))
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
lz4.CompressBlockHC(pg1661, buf, 16)
}
}
func BenchmarkUncompress(b *testing.B) {
buf := make([]byte, len(pg1661))
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
lz4.UncompressBlock(pg1661LZ4, buf)
}
}
func mustLoadFile(f string) []byte {
b, err := ioutil.ReadFile(f)
if err != nil {
panic(err)
}
return b
}
var (
pg1661 = mustLoadFile("testdata/pg1661.txt")
digits = mustLoadFile("testdata/e.txt")
twain = mustLoadFile("testdata/Mark.Twain-Tom.Sawyer.txt")
random = mustLoadFile("testdata/random.data")
pg1661LZ4 = mustLoadFile("testdata/pg1661.txt.lz4")
digitsLZ4 = mustLoadFile("testdata/e.txt.lz4")
twainLZ4 = mustLoadFile("testdata/Mark.Twain-Tom.Sawyer.txt.lz4")
randomLZ4 = mustLoadFile("testdata/random.data.lz4")
)
func benchmarkUncompress(b *testing.B, compressed []byte) {
r := bytes.NewReader(compressed)
zr := lz4.NewReader(r)
// Determine the uncompressed size of testfile.
uncompressedSize, err := io.Copy(ioutil.Discard, zr)
if err != nil {
b.Fatal(err)
}
b.SetBytes(uncompressedSize)
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
r.Reset(compressed)
zr.Reset(r)
io.Copy(ioutil.Discard, zr)
}
}
func BenchmarkUncompressPg1661(b *testing.B) { benchmarkUncompress(b, pg1661LZ4) }
func BenchmarkUncompressDigits(b *testing.B) { benchmarkUncompress(b, digitsLZ4) }
func BenchmarkUncompressTwain(b *testing.B) { benchmarkUncompress(b, twainLZ4) }
func BenchmarkUncompressRand(b *testing.B) { benchmarkUncompress(b, randomLZ4) }
func benchmarkCompress(b *testing.B, uncompressed []byte) {
w := bytes.NewBuffer(nil)
zw := lz4.NewWriter(w)
r := bytes.NewReader(uncompressed)
// Determine the compressed size of testfile.
compressedSize, err := io.Copy(zw, r)
if err != nil {
b.Fatal(err)
}
if err := zw.Close(); err != nil {
b.Fatal(err)
}
b.SetBytes(compressedSize)
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
r.Reset(uncompressed)
zw.Reset(w)
io.Copy(zw, r)
}
}
func BenchmarkCompressPg1661(b *testing.B) { benchmarkCompress(b, pg1661) }
func BenchmarkCompressDigits(b *testing.B) { benchmarkCompress(b, digits) }
func BenchmarkCompressTwain(b *testing.B) { benchmarkCompress(b, twain) }
func BenchmarkCompressRand(b *testing.B) { benchmarkCompress(b, random) }

View File

@ -0,0 +1,397 @@
package lz4
import (
"encoding/binary"
"errors"
)
var (
// ErrInvalidSourceShortBuffer is returned by UncompressBlock or CompressBLock when a compressed
// block is corrupted or the destination buffer is not large enough for the uncompressed data.
ErrInvalidSourceShortBuffer = errors.New("lz4: invalid source or destination buffer too short")
// ErrInvalid is returned when reading an invalid LZ4 archive.
ErrInvalid = errors.New("lz4: bad magic number")
)
// blockHash hashes 4 bytes into a value < winSize.
func blockHash(x uint32) uint32 {
const hasher uint32 = 2654435761 // Knuth multiplicative hash.
return x * hasher >> hashShift
}
// CompressBlockBound returns the maximum size of a given buffer of size n, when not compressible.
func CompressBlockBound(n int) int {
return n + n/255 + 16
}
// UncompressBlock uncompresses the source buffer into the destination one,
// and returns the uncompressed size.
//
// The destination buffer must be sized appropriately.
//
// An error is returned if the source data is invalid or the destination buffer is too small.
func UncompressBlock(src, dst []byte) (si int, err error) {
defer func() {
// It is now faster to let the runtime panic and recover on out of bound slice access
// than checking indices as we go along.
if recover() != nil {
err = ErrInvalidSourceShortBuffer
}
}()
sn := len(src)
if sn == 0 {
return 0, nil
}
var di int
for {
// Literals and match lengths (token).
b := int(src[si])
si++
// Literals.
if lLen := b >> 4; lLen > 0 {
if lLen == 0xF {
for src[si] == 0xFF {
lLen += 0xFF
si++
}
lLen += int(src[si])
si++
}
i := si
si += lLen
di += copy(dst[di:], src[i:si])
if si >= sn {
return di, nil
}
}
si++
_ = src[si] // Bound check elimination.
offset := int(src[si-1]) | int(src[si])<<8
si++
// Match.
mLen := b & 0xF
if mLen == 0xF {
for src[si] == 0xFF {
mLen += 0xFF
si++
}
mLen += int(src[si])
si++
}
mLen += minMatch
// Copy the match.
i := di - offset
if offset > 0 && mLen >= offset {
// Efficiently copy the match dst[di-offset:di] into the dst slice.
bytesToCopy := offset * (mLen / offset)
expanded := dst[i:]
for n := offset; n <= bytesToCopy+offset; n *= 2 {
copy(expanded[n:], expanded[:n])
}
di += bytesToCopy
mLen -= bytesToCopy
}
di += copy(dst[di:], dst[i:i+mLen])
}
}
// CompressBlock compresses the source buffer into the destination one.
// This is the fast version of LZ4 compression and also the default one.
// The size of hashTable must be at least 64Kb.
//
// The size of the compressed data is returned. If it is 0 and no error, then the data is incompressible.
//
// An error is returned if the destination buffer is too small.
func CompressBlock(src, dst []byte, hashTable []int) (di int, err error) {
defer func() {
if recover() != nil {
err = ErrInvalidSourceShortBuffer
}
}()
sn, dn := len(src)-mfLimit, len(dst)
if sn <= 0 || dn == 0 {
return 0, nil
}
var si int
// Fast scan strategy: the hash table only stores the last 4 bytes sequences.
// const accInit = 1 << skipStrength
anchor := si // Position of the current literals.
// acc := accInit // Variable step: improves performance on non-compressible data.
for si < sn {
// Hash the next 4 bytes (sequence)...
match := binary.LittleEndian.Uint32(src[si:])
h := blockHash(match)
ref := hashTable[h]
hashTable[h] = si
if ref >= sn { // Invalid reference (dirty hashtable).
si++
continue
}
offset := si - ref
if offset <= 0 || offset >= winSize || // Out of window.
match != binary.LittleEndian.Uint32(src[ref:]) { // Hash collision on different matches.
// si += acc >> skipStrength
// acc++
si++
continue
}
// Match found.
// acc = accInit
lLen := si - anchor // Literal length.
// Encode match length part 1.
si += minMatch
mLen := si // Match length has minMatch already.
// Find the longest match, first looking by batches of 8 bytes.
for si < sn && binary.LittleEndian.Uint64(src[si:]) == binary.LittleEndian.Uint64(src[si-offset:]) {
si += 8
}
// Then byte by byte.
for si < sn && src[si] == src[si-offset] {
si++
}
mLen = si - mLen
if mLen < 0xF {
dst[di] = byte(mLen)
} else {
dst[di] = 0xF
}
// Encode literals length.
if lLen < 0xF {
dst[di] |= byte(lLen << 4)
} else {
dst[di] |= 0xF0
di++
l := lLen - 0xF
for ; l >= 0xFF; l -= 0xFF {
dst[di] = 0xFF
di++
}
dst[di] = byte(l)
}
di++
// Literals.
copy(dst[di:], src[anchor:anchor+lLen])
di += lLen + 2
anchor = si
// Encode offset.
_ = dst[di] // Bound check elimination.
dst[di-2], dst[di-1] = byte(offset), byte(offset>>8)
// Encode match length part 2.
if mLen >= 0xF {
for mLen -= 0xF; mLen >= 0xFF; mLen -= 0xFF {
dst[di] = 0xFF
di++
}
dst[di] = byte(mLen)
di++
}
}
if anchor == 0 {
// Incompressible.
return 0, nil
}
// Last literals.
lLen := len(src) - anchor
if lLen < 0xF {
dst[di] = byte(lLen << 4)
} else {
dst[di] = 0xF0
di++
for lLen -= 0xF; lLen >= 0xFF; lLen -= 0xFF {
dst[di] = 0xFF
di++
}
dst[di] = byte(lLen)
}
di++
// Write the last literals.
if di >= anchor {
// Incompressible.
return 0, nil
}
di += copy(dst[di:], src[anchor:])
return di, nil
}
// CompressBlockHC compresses the source buffer src into the destination dst
// with max search depth (use 0 or negative value for no max).
//
// CompressBlockHC compression ratio is better than CompressBlock but it is also slower.
//
// The size of the compressed data is returned. If it is 0 and no error, then the data is not compressible.
//
// An error is returned if the destination buffer is too small.
func CompressBlockHC(src, dst []byte, depth int) (di int, err error) {
defer func() {
if recover() != nil {
err = ErrInvalidSourceShortBuffer
}
}()
sn, dn := len(src)-mfLimit, len(dst)
if sn <= 0 || dn == 0 {
return 0, nil
}
var si int
// hashTable: stores the last position found for a given hash
// chaingTable: stores previous positions for a given hash
var hashTable, chainTable [winSize]int
if depth <= 0 {
depth = winSize
}
anchor := si
for si < sn {
// Hash the next 4 bytes (sequence).
match := binary.LittleEndian.Uint32(src[si:])
h := blockHash(match)
// Follow the chain until out of window and give the longest match.
mLen := 0
offset := 0
for next, try := hashTable[h], depth; try > 0 && next > 0 && si-next < winSize; next = chainTable[next&winMask] {
// The first (mLen==0) or next byte (mLen>=minMatch) at current match length
// must match to improve on the match length.
if src[next+mLen] != src[si+mLen] {
continue
}
ml := 0
// Compare the current position with a previous with the same hash.
for ml < sn-si && binary.LittleEndian.Uint64(src[next+ml:]) == binary.LittleEndian.Uint64(src[si+ml:]) {
ml += 8
}
for ml < sn-si && src[next+ml] == src[si+ml] {
ml++
}
if ml+1 < minMatch || ml <= mLen {
// Match too small (<minMath) or smaller than the current match.
continue
}
// Found a longer match, keep its position and length.
mLen = ml
offset = si - next
// Try another previous position with the same hash.
try--
}
chainTable[si&winMask] = hashTable[h]
hashTable[h] = si
// No match found.
if mLen == 0 {
si++
continue
}
// Match found.
// Update hash/chain tables with overlapping bytes:
// si already hashed, add everything from si+1 up to the match length.
winStart := si + 1
if ws := si + mLen - winSize; ws > winStart {
winStart = ws
}
for si, ml := winStart, si+mLen; si < ml; {
match >>= 8
match |= uint32(src[si+3]) << 24
h := blockHash(match)
chainTable[si&winMask] = hashTable[h]
hashTable[h] = si
si++
}
lLen := si - anchor
si += mLen
mLen -= minMatch // Match length does not include minMatch.
if mLen < 0xF {
dst[di] = byte(mLen)
} else {
dst[di] = 0xF
}
// Encode literals length.
if lLen < 0xF {
dst[di] |= byte(lLen << 4)
} else {
dst[di] |= 0xF0
di++
l := lLen - 0xF
for ; l >= 0xFF; l -= 0xFF {
dst[di] = 0xFF
di++
}
dst[di] = byte(l)
}
di++
// Literals.
copy(dst[di:], src[anchor:anchor+lLen])
di += lLen
anchor = si
// Encode offset.
di += 2
dst[di-2], dst[di-1] = byte(offset), byte(offset>>8)
// Encode match length part 2.
if mLen >= 0xF {
for mLen -= 0xF; mLen >= 0xFF; mLen -= 0xFF {
dst[di] = 0xFF
di++
}
dst[di] = byte(mLen)
di++
}
}
if anchor == 0 {
// Incompressible.
return 0, nil
}
// Last literals.
lLen := len(src) - anchor
if lLen < 0xF {
dst[di] = byte(lLen << 4)
} else {
dst[di] = 0xF0
di++
lLen -= 0xF
for ; lLen >= 0xFF; lLen -= 0xFF {
dst[di] = 0xFF
di++
}
dst[di] = byte(lLen)
}
di++
// Write the last literals.
if di >= anchor {
// Incompressible.
return 0, nil
}
di += copy(dst[di:], src[anchor:])
return di, nil
}

View File

@ -0,0 +1,98 @@
//+build go1.9
package lz4_test
import (
"fmt"
"io/ioutil"
"reflect"
"testing"
"gitee.com/johng/gf/third/github.com/pierrec/lz4"
)
type testcase struct {
file string
compressible bool
src []byte
}
var rawFiles = []testcase{
// {"testdata/207326ba-36f8-11e7-954a-aca46ba8ca73.png", true, nil},
{"testdata/e.txt", true, nil},
{"testdata/gettysburg.txt", true, nil},
{"testdata/Mark.Twain-Tom.Sawyer.txt", true, nil},
{"testdata/pg1661.txt", true, nil},
{"testdata/pi.txt", true, nil},
{"testdata/random.data", false, nil},
{"testdata/repeat.txt", true, nil},
}
func TestCompressUncompressBlock(t *testing.T) {
type compressor func(s, d []byte) (int, error)
run := func(tc testcase, compress compressor) int {
t.Helper()
src := tc.src
// Compress the data.
zbuf := make([]byte, lz4.CompressBlockBound(len(src)))
n, err := compress(src, zbuf)
if err != nil {
t.Error(err)
return 0
}
zbuf = zbuf[:n]
// Make sure that it was actually compressed unless not compressible.
if !tc.compressible {
return 0
}
if n == 0 || n >= len(src) {
t.Errorf("data not compressed: %d/%d", n, len(src))
return 0
}
// Uncompress the data.
buf := make([]byte, len(src))
n, err = lz4.UncompressBlock(zbuf, buf)
if err != nil {
t.Fatal(err)
}
buf = buf[:n]
if !reflect.DeepEqual(src, buf) {
t.Error("uncompressed compressed data not matching initial input")
return 0
}
return len(zbuf)
}
for _, tc := range rawFiles {
src, err := ioutil.ReadFile(tc.file)
if err != nil {
t.Fatal(err)
}
tc.src = src
var n, nhc int
t.Run("", func(t *testing.T) {
tc := tc
t.Run(tc.file, func(t *testing.T) {
t.Parallel()
n = run(tc, func(src, dst []byte) (int, error) {
var ht [1 << 16]int
return lz4.CompressBlock(src, dst, ht[:])
})
})
t.Run(fmt.Sprintf("%s HC", tc.file), func(t *testing.T) {
t.Parallel()
nhc = run(tc, func(src, dst []byte) (int, error) {
return lz4.CompressBlockHC(src, dst, -1)
})
})
})
fmt.Printf("%-40s: %8d / %8d / %8d\n", tc.file, n, nhc, len(src))
}
}

View File

@ -0,0 +1,21 @@
// +build lz4debug
package lz4
import (
"fmt"
"os"
"path/filepath"
"runtime"
)
func debug(args ...interface{}) {
_, file, line, _ := runtime.Caller(1)
file = filepath.Base(file)
f := fmt.Sprintf("LZ4: %s:%d %s", file, line, args[0])
if f[len(f)-1] != '\n' {
f += "\n"
}
fmt.Fprintf(os.Stderr, f, args[1:]...)
}

View File

@ -0,0 +1,5 @@
// +build !lz4debug
package lz4
func debug(args ...interface{}) {}

View File

@ -0,0 +1,13 @@
// Expose some internals for testing purposes
package lz4
// expose the possible block max sizes
var BlockMaxSizeItems []int
func init() {
for s := range bsMapValue {
BlockMaxSizeItems = append(BlockMaxSizeItems, s)
}
}
var FrameSkipMagic = frameSkipMagic

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

View File

@ -0,0 +1 @@
"MM@"<22><><EFBFBD><EFBFBD>p+[<5B>

Some files were not shown because too many files have changed in this diff Show More