From 74be9fac1863b7fbb389db3182aaafa6a3c261b5 Mon Sep 17 00:00:00 2001 From: John Date: Sun, 15 Mar 2020 19:32:26 +0800 Subject: [PATCH] add file rotation feature for package glog; improve gpool/gfpool; fix issue in gfile.MTimeMillisecond --- container/garray/garray_sorted_any.go | 6 +- database/gdb/gdb_core.go | 2 +- database/gdb/gdb_unit_init_test.go | 1 + encoding/gcompress/gcompress_gzip.go | 60 ++++++- .../gcompress/gcompress_z_unit_gzip_test.go | 42 ++++- ...gcompress_zip_file.go => gcompress_zip.go} | 0 encoding/gcompress/testdata/gzip/file.txt | 1 + os/gfile/gfile.go | 4 +- os/gfile/gfile_time.go | 2 +- os/gfpool/gfpool.go | 145 +-------------- os/gfpool/gfpool_file.go | 55 ++++++ os/gfpool/gfpool_pool.go | 121 +++++++++++++ os/gfpool/gfpool_z_bench_test.go | 42 +++-- os/glog/glog_logger.go | 39 +++- os/glog/glog_logger_config.go | 45 +++-- os/glog/glog_logger_rotate.go | 168 ++++++++++++++++++ os/glog/glog_z_unit_rotate_test.go | 56 ++++++ 17 files changed, 593 insertions(+), 196 deletions(-) rename encoding/gcompress/{gcompress_zip_file.go => gcompress_zip.go} (100%) create mode 100644 encoding/gcompress/testdata/gzip/file.txt create mode 100644 os/gfpool/gfpool_file.go create mode 100644 os/gfpool/gfpool_pool.go create mode 100644 os/glog/glog_logger_rotate.go create mode 100644 os/glog/glog_z_unit_rotate_test.go diff --git a/container/garray/garray_sorted_any.go b/container/garray/garray_sorted_any.go index 32439e8f8..b55dca332 100644 --- a/container/garray/garray_sorted_any.go +++ b/container/garray/garray_sorted_any.go @@ -33,9 +33,9 @@ type SortedArray struct { // NewSortedArray creates and returns an empty sorted array. // The parameter is used to specify whether using array in concurrent-safety, which is false in default. // The parameter used to compare values to sort in array, -// if it returns value < 0, means v1 < v2; -// if it returns value = 0, means v1 = v2; -// if it returns value > 0, means v1 > v2; +// if it returns value < 0, means v1 < v2; the v1 will be inserted before v2; +// if it returns value = 0, means v1 = v2; the v1 will be replaced by v2; +// if it returns value > 0, means v1 > v2; the v1 will be inserted after v2; func NewSortedArray(comparator func(a, b interface{}) int, safe ...bool) *SortedArray { return NewSortedArraySize(0, comparator, safe...) } diff --git a/database/gdb/gdb_core.go b/database/gdb/gdb_core.go index 92e2ead29..7770f2cdd 100644 --- a/database/gdb/gdb_core.go +++ b/database/gdb/gdb_core.go @@ -721,7 +721,7 @@ func (c *Core) MarshalJSON() ([]byte, error) { // writeSqlToLogger outputs the sql object to logger. // It is enabled when configuration "debug" is true. func (c *Core) writeSqlToLogger(v *Sql) { - s := fmt.Sprintf("[%d ms] %s", v.End-v.Start, v.Format) + s := fmt.Sprintf("[%3d ms] %s", v.End-v.Start, v.Format) if v.Error != nil { s += "\nError: " + v.Error.Error() c.logger.StackWithFilter(gPATH_FILTER_KEY).Error(s) diff --git a/database/gdb/gdb_unit_init_test.go b/database/gdb/gdb_unit_init_test.go index 2879fe6ab..957529bd1 100644 --- a/database/gdb/gdb_unit_init_test.go +++ b/database/gdb/gdb_unit_init_test.go @@ -62,6 +62,7 @@ func init() { } else { db = r } + db.SetDebug(true) schemaTemplate := "CREATE DATABASE IF NOT EXISTS `%s` CHARACTER SET UTF8" if _, err := db.Exec(fmt.Sprintf(schemaTemplate, SCHEMA1)); err != nil { gtest.Error(err) diff --git a/encoding/gcompress/gcompress_gzip.go b/encoding/gcompress/gcompress_gzip.go index 43bd05c43..fd5cc90a3 100644 --- a/encoding/gcompress/gcompress_gzip.go +++ b/encoding/gcompress/gcompress_gzip.go @@ -9,10 +9,11 @@ package gcompress import ( "bytes" "compress/gzip" + "github.com/gogf/gf/os/gfile" "io" ) -// Gzip compresses with gzip algorithm. +// Gzip compresses using gzip algorithm. // The optional parameter specifies the compression level from // 1 to 9 which means from none to the best compression. // @@ -38,6 +39,38 @@ func Gzip(data []byte, level ...int) ([]byte, error) { return buf.Bytes(), nil } +// GzipFile compresses the file to using gzip algorithm. +func GzipFile(src, dst string, level ...int) error { + var writer *gzip.Writer + var err error + srcFile, err := gfile.Open(src) + if err != nil { + return err + } + defer srcFile.Close() + dstFile, err := gfile.Create(dst) + if err != nil { + return err + } + defer dstFile.Close() + + if len(level) > 0 { + writer, err = gzip.NewWriterLevel(dstFile, level[0]) + if err != nil { + return err + } + } else { + writer = gzip.NewWriter(dstFile) + } + defer writer.Close() + + _, err = io.Copy(writer, srcFile) + if err != nil { + return err + } + return nil +} + // UnGzip decompresses with gzip algorithm. func UnGzip(data []byte) ([]byte, error) { var buf bytes.Buffer @@ -53,3 +86,28 @@ func UnGzip(data []byte) ([]byte, error) { } return buf.Bytes(), nil } + +// UnGzip decompresses file to using gzip algorithm. +func UnGzipFile(src, dst string) error { + srcFile, err := gfile.Open(src) + if err != nil { + return err + } + defer srcFile.Close() + dstFile, err := gfile.Create(dst) + if err != nil { + return err + } + defer dstFile.Close() + + reader, err := gzip.NewReader(srcFile) + if err != nil { + return err + } + defer reader.Close() + + if _, err = io.Copy(dstFile, reader); err != nil { + return err + } + return nil +} diff --git a/encoding/gcompress/gcompress_z_unit_gzip_test.go b/encoding/gcompress/gcompress_z_unit_gzip_test.go index 712261a03..14ffab44a 100644 --- a/encoding/gcompress/gcompress_z_unit_gzip_test.go +++ b/encoding/gcompress/gcompress_z_unit_gzip_test.go @@ -7,6 +7,9 @@ package gcompress_test import ( + "github.com/gogf/gf/debug/gdebug" + "github.com/gogf/gf/os/gfile" + "github.com/gogf/gf/os/gtime" "testing" "github.com/gogf/gf/encoding/gcompress" @@ -26,14 +29,37 @@ func Test_Gzip_UnGzip(t *testing.T) { 0x24, 0xa8, 0xd1, 0x0d, 0x00, 0x00, 0x00, } + gtest.Case(t, func() { + arr := []byte(src) + data, _ := gcompress.Gzip(arr) + gtest.Assert(data, gzip) - arr := []byte(src) - data, _ := gcompress.Gzip(arr) - gtest.Assert(data, gzip) + data, _ = gcompress.UnGzip(gzip) + gtest.Assert(data, arr) - data, _ = gcompress.UnGzip(gzip) - gtest.Assert(data, arr) - - data, _ = gcompress.UnGzip(gzip[1:]) - gtest.Assert(data, nil) + data, _ = gcompress.UnGzip(gzip[1:]) + gtest.Assert(data, nil) + }) +} + +func Test_Gzip_UnGzip_File(t *testing.T) { + srcPath := gfile.Join(gdebug.CallerDirectory(), "testdata", "gzip", "file.txt") + dstPath1 := gfile.Join(gfile.TempDir(), gtime.TimestampNanoStr(), "gzip.zip") + dstPath2 := gfile.Join(gfile.TempDir(), gtime.TimestampNanoStr(), "file.txt") + + // Compress. + gtest.Case(t, func() { + err := gcompress.GzipFile(srcPath, dstPath1, 9) + gtest.Assert(err, nil) + defer gfile.Remove(dstPath1) + gtest.Assert(gfile.Exists(dstPath1), true) + + // Decompress. + err = gcompress.UnGzipFile(dstPath1, dstPath2) + gtest.Assert(err, nil) + defer gfile.Remove(dstPath2) + gtest.Assert(gfile.Exists(dstPath2), true) + + gtest.Assert(gfile.GetContents(srcPath), gfile.GetContents(dstPath2)) + }) } diff --git a/encoding/gcompress/gcompress_zip_file.go b/encoding/gcompress/gcompress_zip.go similarity index 100% rename from encoding/gcompress/gcompress_zip_file.go rename to encoding/gcompress/gcompress_zip.go diff --git a/encoding/gcompress/testdata/gzip/file.txt b/encoding/gcompress/testdata/gzip/file.txt new file mode 100644 index 000000000..14a9d9b49 --- /dev/null +++ b/encoding/gcompress/testdata/gzip/file.txt @@ -0,0 +1 @@ +This is a test file for gzip compression. \ No newline at end of file diff --git a/os/gfile/gfile.go b/os/gfile/gfile.go index d905f09d4..f230bbad0 100644 --- a/os/gfile/gfile.go +++ b/os/gfile/gfile.go @@ -312,7 +312,7 @@ func SelfDir() string { return filepath.Dir(SelfPath()) } -// Basename returns the last element of path. +// Basename returns the last element of path, which contains file extension. // Trailing path separators are removed before extracting the last element. // If the path is empty, Base returns ".". // If the path consists entirely of separators, Basename returns a single separator. @@ -320,7 +320,7 @@ func Basename(path string) string { return filepath.Base(path) } -// Name returns the last element of path without extension. +// Name returns the last element of path without file extension. func Name(path string) string { base := filepath.Base(path) if i := strings.LastIndexByte(base, '.'); i != -1 { diff --git a/os/gfile/gfile_time.go b/os/gfile/gfile_time.go index b468bc712..8cacb7677 100644 --- a/os/gfile/gfile_time.go +++ b/os/gfile/gfile_time.go @@ -25,5 +25,5 @@ func MTimeMillisecond(path string) int64 { if e != nil { return 0 } - return int64(s.ModTime().Nanosecond() / 1000000) + return s.ModTime().UnixNano() / 1000000 } diff --git a/os/gfpool/gfpool.go b/os/gfpool/gfpool.go index 9dd92af0c..db4e7ecd9 100644 --- a/os/gfpool/gfpool.go +++ b/os/gfpool/gfpool.go @@ -1,4 +1,4 @@ -// Copyright 2017-2019 gf Author(https://github.com/gogf/gf). All Rights Reserved. +// Copyright 2017-2020 gf Author(https://github.com/gogf/gf). All Rights Reserved. // // This Source Code Form is subject to the terms of the MIT License. // If a copy of the MIT was not distributed with this file, @@ -8,15 +8,11 @@ package gfpool import ( - "fmt" - "github.com/gogf/gf/os/gfile" - "os" - "time" - "github.com/gogf/gf/container/gmap" "github.com/gogf/gf/container/gpool" "github.com/gogf/gf/container/gtype" - "github.com/gogf/gf/os/gfsnotify" + "os" + "time" ) // File pointer pool. @@ -30,6 +26,7 @@ type Pool struct { // File is an item in the pool. type File struct { *os.File // Underlying file pointer. + stat os.FileInfo // State of current file pointer. pid int // Belonging pool id, which is set when file pointer created. It's used to check whether the pool is recreated. pool *Pool // Belonging ool. flag int // Flash for opening file. @@ -41,137 +38,3 @@ var ( // Global file pointer pool. pools = gmap.NewStrAnyMap(true) ) - -// Open creates and returns a file item with given file path, flag and opening permission. -// It automatically creates an associated file pointer pool internally when it's called first time. -// It retrieves a file item from the file pointer pool after then. -func Open(path string, flag int, perm os.FileMode, ttl ...time.Duration) (file *File, err error) { - var fpTTL time.Duration - if len(ttl) > 0 { - fpTTL = ttl[0] - } - path, err = gfile.Search(path) - if err != nil { - return nil, err - } - pool := pools.GetOrSetFuncLock( - fmt.Sprintf("%s&%d&%d&%d", path, flag, fpTTL, perm), - func() interface{} { - return New(path, flag, perm, fpTTL) - }, - ).(*Pool) - - return pool.File() -} - -// New creates and returns a file pointer pool with given file path, flag and opening permission. -// -// Note the expiration logic: -// ttl = 0 : not expired; -// ttl < 0 : immediate expired after use; -// ttl > 0 : timeout expired; -// It is not expired in default. -func New(path string, flag int, perm os.FileMode, ttl ...time.Duration) *Pool { - var fpTTL time.Duration - if len(ttl) > 0 { - fpTTL = ttl[0] - } - p := &Pool{ - id: gtype.NewInt(), - ttl: fpTTL, - init: gtype.NewBool(), - } - p.pool = newFilePool(p, path, flag, perm, fpTTL) - return p -} - -// newFilePool creates and returns a file pointer pool with given file path, flag and opening permission. -func newFilePool(p *Pool, path string, flag int, perm os.FileMode, ttl time.Duration) *gpool.Pool { - pool := gpool.New(ttl, func() (interface{}, error) { - file, err := os.OpenFile(path, flag, perm) - if err != nil { - return nil, err - } - return &File{ - File: file, - pool: p, - pid: p.id.Val(), - flag: flag, - perm: perm, - path: path, - }, nil - }, func(i interface{}) { - _ = i.(*File).File.Close() - }) - return pool -} - -// File retrieves file item from the file pointer pool and returns it. It creates one if -// the file pointer pool is empty. -// Note that it should be closed when it will never be used. When it's closed, it is not -// really closed the underlying file pointer but put back to the file pinter pool. -func (p *Pool) File() (*File, error) { - if v, err := p.pool.Get(); err != nil { - return nil, err - } else { - f := v.(*File) - stat, err := os.Stat(f.path) - if f.flag&os.O_CREATE > 0 { - if os.IsNotExist(err) { - if file, err := os.OpenFile(f.path, f.flag, f.perm); err != nil { - return nil, err - } else { - f.File = file - if stat, err = f.Stat(); err != nil { - return nil, err - } - } - } - } - if f.flag&os.O_TRUNC > 0 { - if stat.Size() > 0 { - if err := f.Truncate(0); err != nil { - return nil, err - } - } - } - if f.flag&os.O_APPEND > 0 { - if _, err := f.Seek(0, 2); err != nil { - return nil, err - } - } else { - if _, err := f.Seek(0, 0); err != nil { - return nil, err - } - } - // It firstly checks using !p.init.Val() for performance purpose. - if !p.init.Val() && p.init.Cas(false, true) { - _, _ = gfsnotify.Add(f.path, func(event *gfsnotify.Event) { - // If teh file is removed or renamed, recreates the pool by increasing the pool id. - if event.IsRemove() || event.IsRename() { - // It drops the old pool. - p.id.Add(1) - // Clears the pool items staying in the pool. - p.pool.Clear() - // It uses another adding to drop the file items between the two adding. - // Whenever the pool id changes, the pool will be recreated. - p.id.Add(1) - } - }, false) - } - return f, nil - } -} - -// Close closes current file pointer pool. -func (p *Pool) Close() { - p.pool.Close() -} - -// Close puts the file pointer back to the file pointer pool. -func (f *File) Close() error { - if f.pid == f.pool.id.Val() { - return f.pool.pool.Put(f) - } - return nil -} diff --git a/os/gfpool/gfpool_file.go b/os/gfpool/gfpool_file.go new file mode 100644 index 000000000..b985722b1 --- /dev/null +++ b/os/gfpool/gfpool_file.go @@ -0,0 +1,55 @@ +// Copyright 2017-2020 gf Author(https://github.com/gogf/gf). All Rights Reserved. +// +// This Source Code Form is subject to the terms of the MIT License. +// If a copy of the MIT was not distributed with this file, +// You can obtain one at https://github.com/gogf/gf. + +package gfpool + +import ( + "errors" + "fmt" + "os" + "time" +) + +// Open creates and returns a file item with given file path, flag and opening permission. +// It automatically creates an associated file pointer pool internally when it's called first time. +// It retrieves a file item from the file pointer pool after then. +func Open(path string, flag int, perm os.FileMode, ttl ...time.Duration) (file *File, err error) { + var fpTTL time.Duration + if len(ttl) > 0 { + fpTTL = ttl[0] + } + // DO NOT search the path here wasting performance! + // Leave following codes just for warning you. + // + //path, err = gfile.Search(path) + //if err != nil { + // return nil, err + //} + pool := pools.GetOrSetFuncLock( + fmt.Sprintf("%s&%d&%d&%d", path, flag, fpTTL, perm), + func() interface{} { + return New(path, flag, perm, fpTTL) + }, + ).(*Pool) + + return pool.File() +} + +// Stat returns the FileInfo structure describing file. +func (f *File) Stat() (os.FileInfo, error) { + if f.stat == nil { + return nil, errors.New("file stat is empty") + } + return f.stat, nil +} + +// Close puts the file pointer back to the file pointer pool. +func (f *File) Close() error { + if f.pid == f.pool.id.Val() { + return f.pool.pool.Put(f) + } + return nil +} diff --git a/os/gfpool/gfpool_pool.go b/os/gfpool/gfpool_pool.go new file mode 100644 index 000000000..e83abf6aa --- /dev/null +++ b/os/gfpool/gfpool_pool.go @@ -0,0 +1,121 @@ +// Copyright 2017-2020 gf Author(https://github.com/gogf/gf). All Rights Reserved. +// +// This Source Code Form is subject to the terms of the MIT License. +// If a copy of the MIT was not distributed with this file, +// You can obtain one at https://github.com/gogf/gf. + +package gfpool + +import ( + "os" + "time" + + "github.com/gogf/gf/container/gpool" + "github.com/gogf/gf/container/gtype" + "github.com/gogf/gf/os/gfsnotify" +) + +// New creates and returns a file pointer pool with given file path, flag and opening permission. +// +// Note the expiration logic: +// ttl = 0 : not expired; +// ttl < 0 : immediate expired after use; +// ttl > 0 : timeout expired; +// It is not expired in default. +func New(path string, flag int, perm os.FileMode, ttl ...time.Duration) *Pool { + var fpTTL time.Duration + if len(ttl) > 0 { + fpTTL = ttl[0] + } + p := &Pool{ + id: gtype.NewInt(), + ttl: fpTTL, + init: gtype.NewBool(), + } + p.pool = newFilePool(p, path, flag, perm, fpTTL) + return p +} + +// newFilePool creates and returns a file pointer pool with given file path, flag and opening permission. +func newFilePool(p *Pool, path string, flag int, perm os.FileMode, ttl time.Duration) *gpool.Pool { + pool := gpool.New(ttl, func() (interface{}, error) { + file, err := os.OpenFile(path, flag, perm) + if err != nil { + return nil, err + } + return &File{ + File: file, + pid: p.id.Val(), + pool: p, + flag: flag, + perm: perm, + path: path, + }, nil + }, func(i interface{}) { + _ = i.(*File).File.Close() + }) + return pool +} + +// File retrieves file item from the file pointer pool and returns it. It creates one if +// the file pointer pool is empty. +// Note that it should be closed when it will never be used. When it's closed, it is not +// really closed the underlying file pointer but put back to the file pinter pool. +func (p *Pool) File() (*File, error) { + if v, err := p.pool.Get(); err != nil { + return nil, err + } else { + var err error + f := v.(*File) + f.stat, err = os.Stat(f.path) + if f.flag&os.O_CREATE > 0 { + if os.IsNotExist(err) { + if f.File, err = os.OpenFile(f.path, f.flag, f.perm); err != nil { + return nil, err + } else { + // Retrieve the state of the new created file. + if f.stat, err = f.File.Stat(); err != nil { + return nil, err + } + } + } + } + if f.flag&os.O_TRUNC > 0 { + if f.stat.Size() > 0 { + if err = f.Truncate(0); err != nil { + return nil, err + } + } + } + if f.flag&os.O_APPEND > 0 { + if _, err = f.Seek(0, 2); err != nil { + return nil, err + } + } else { + if _, err = f.Seek(0, 0); err != nil { + return nil, err + } + } + // It firstly checks using !p.init.Val() for performance purpose. + if !p.init.Val() && p.init.Cas(false, true) { + _, _ = gfsnotify.Add(f.path, func(event *gfsnotify.Event) { + // If teh file is removed or renamed, recreates the pool by increasing the pool id. + if event.IsRemove() || event.IsRename() { + // It drops the old pool. + p.id.Add(1) + // Clears the pool items staying in the pool. + p.pool.Clear() + // It uses another adding to drop the file items between the two adding. + // Whenever the pool id changes, the pool will be recreated. + p.id.Add(1) + } + }, false) + } + return f, nil + } +} + +// Close closes current file pointer pool. +func (p *Pool) Close() { + p.pool.Close() +} diff --git a/os/gfpool/gfpool_z_bench_test.go b/os/gfpool/gfpool_z_bench_test.go index da59aa54f..025865033 100644 --- a/os/gfpool/gfpool_z_bench_test.go +++ b/os/gfpool/gfpool_z_bench_test.go @@ -5,44 +5,62 @@ import ( "testing" ) -func Benchmark_os_Open_Close_ALLFlags(b *testing.B) { +func Benchmark_OS_Open_Close_ALLFlags(b *testing.B) { for i := 0; i < b.N; i++ { - f, _ := os.OpenFile("/tmp/bench-test", os.O_RDWR|os.O_CREATE|os.O_TRUNC|os.O_APPEND, 0666) + f, err := os.OpenFile("/tmp/bench-test", os.O_RDWR|os.O_CREATE|os.O_TRUNC|os.O_APPEND, 0666) + if err != nil { + panic(err) + } f.Close() } } -func Benchmark_gfpool_Open_Close_ALLFlags(b *testing.B) { +func Benchmark_GFPool_Open_Close_ALLFlags(b *testing.B) { for i := 0; i < b.N; i++ { - f, _ := Open("/tmp/bench-test", os.O_RDWR|os.O_CREATE|os.O_TRUNC|os.O_APPEND, 0666) + f, err := Open("/tmp/bench-test", os.O_RDWR|os.O_CREATE|os.O_TRUNC|os.O_APPEND, 0666) + if err != nil { + panic(err) + } f.Close() } } -func Benchmark_os_Open_Close_RDWR(b *testing.B) { +func Benchmark_OS_Open_Close_RDWR(b *testing.B) { for i := 0; i < b.N; i++ { - f, _ := os.OpenFile("/tmp/bench-test", os.O_RDWR, 0666) + f, err := os.OpenFile("/tmp/bench-test", os.O_RDWR, 0666) + if err != nil { + panic(err) + } f.Close() } } -func Benchmark_gfpool_Open_Close_RDWR(b *testing.B) { +func Benchmark_GFPool_Open_Close_RDWR(b *testing.B) { for i := 0; i < b.N; i++ { - f, _ := Open("/tmp/bench-test", os.O_RDWR, 0666) + f, err := Open("/tmp/bench-test", os.O_RDWR, 0666) + if err != nil { + panic(err) + } f.Close() } } -func Benchmark_os_Open_Close_RDONLY(b *testing.B) { +func Benchmark_OS_Open_Close_RDONLY(b *testing.B) { for i := 0; i < b.N; i++ { - f, _ := os.OpenFile("/tmp/bench-test", os.O_RDONLY, 0666) + f, err := os.OpenFile("/tmp/bench-test", os.O_RDONLY, 0666) + if err != nil { + panic(err) + } f.Close() } } -func Benchmark_gfpool_Open_Close_RDONLY(b *testing.B) { +func Benchmark_GFPool_Open_Close_RDONLY(b *testing.B) { for i := 0; i < b.N; i++ { - f, _ := Open("/tmp/bench-test", os.O_RDONLY, 0666) + f, err := Open("/tmp/bench-test", os.O_RDONLY, 0666) + if err != nil { + panic(err) + } f.Close() } } diff --git a/os/glog/glog_logger.go b/os/glog/glog_logger.go index 781bb277a..0575ed436 100644 --- a/os/glog/glog_logger.go +++ b/os/glog/glog_logger.go @@ -10,9 +10,11 @@ import ( "bytes" "fmt" "github.com/gogf/gf/internal/intlog" + "github.com/gogf/gf/os/gtimer" "io" "os" "strings" + "sync" "time" "github.com/gogf/gf/debug/gdebug" @@ -26,8 +28,9 @@ import ( // Logger is the struct for logging management. type Logger struct { - parent *Logger // Parent logger. - config Config // Logger configuration. + mu sync.Mutex // Mutex is not for common logging, but for file rotation feature. + parent *Logger // Parent logger. + config Config // Logger configuration. } const ( @@ -50,9 +53,11 @@ const ( // New creates and returns a custom logger. func New() *Logger { - return &Logger{ + logger := &Logger{ config: DefaultConfig(), } + gtimer.AddOnce(time.Second, logger.rotateChecks) + return logger } // NewWithWriter creates and returns a custom logger with io.Writer. @@ -63,6 +68,7 @@ func NewWithWriter(writer io.Writer) *Logger { } // Clone returns a new logger, which is the clone the current logger. +// It's commonly used for chaining operations. func (l *Logger) Clone() *Logger { logger := Logger{} logger = *l @@ -74,10 +80,6 @@ func (l *Logger) Clone() *Logger { // It returns nil if file logging is disabled, or file opening fails. func (l *Logger) getFilePointer() *gfpool.File { if path := l.config.Path; path != "" { - // Content containing "{}" in the file name is formatted using gtime. - file, _ := gregex.ReplaceStringFunc(`{.+?}`, l.config.File, func(s string) string { - return gtime.Now().Format(strings.Trim(s, "{}")) - }) // Create path if it does not exist. if !gfile.Exists(path) { if err := gfile.Mkdir(path); err != nil { @@ -86,7 +88,7 @@ func (l *Logger) getFilePointer() *gfpool.File { } } if fp, err := gfpool.Open( - path+gfile.Separator+file, + l.getFilePath(), gDEFAULT_FILE_POOL_FLAGS, gDEFAULT_FPOOL_PERM, gDEFAULT_FPOOL_EXPIRE); err == nil { @@ -98,6 +100,15 @@ func (l *Logger) getFilePointer() *gfpool.File { return nil } +// getFilePath returns the logging file path. +func (l *Logger) getFilePath() string { + // Content containing "{}" in the file name is formatted using gtime. + file, _ := gregex.ReplaceStringFunc(`{.+?}`, l.config.File, func(s string) string { + return gtime.Now().Format(strings.Trim(s, "{}")) + }) + return gfile.Join(l.config.Path, file) +} + // print prints to defined writer, logging file or passed . func (l *Logger) print(std io.Writer, lead string, value ...interface{}) { buffer := bytes.NewBuffer(nil) @@ -183,6 +194,18 @@ func (l *Logger) printToWriter(std io.Writer, buffer *bytes.Buffer) { if l.config.Writer == nil { if f := l.getFilePointer(); f != nil { defer f.Close() + // Rotation file size checks. + if l.config.RotateSize > 0 { + state, err := f.Stat() + if err != nil { + panic(err) + } + if state.Size() > l.config.RotateSize { + l.rotateFile() + l.printToWriter(std, buffer) + return + } + } if _, err := io.WriteString(f, buffer.String()); err != nil { fmt.Fprintln(os.Stderr, err.Error()) } diff --git a/os/glog/glog_logger_config.go b/os/glog/glog_logger_config.go index c66836d9f..7b8ee2d56 100644 --- a/os/glog/glog_logger_config.go +++ b/os/glog/glog_logger_config.go @@ -14,34 +14,41 @@ import ( "github.com/gogf/gf/util/gutil" "io" "strings" + "time" ) // Config is the configuration object for logger. type Config struct { - Writer io.Writer // Customized io.Writer. - Flags int // Extra flags for logging output features. - Path string // Logging directory path. - File string // Format for logging file. - Level int // Output level. - Prefix string // Prefix string for every logging content. - StSkip int // Skip count for stack. - StStatus int // Stack status(1: enabled - default; 0: disabled) - StFilter string // Stack string filter. - HeaderPrint bool `c:"header"` // Print header or not(true in default). - StdoutPrint bool `c:"stdout"` // Output to stdout or not(true in default). - LevelPrefixes map[int]string // Logging level to its prefix string mapping. + Writer io.Writer // Customized io.Writer. + Flags int // Extra flags for logging output features. + Path string // Logging directory path. + File string // Format for logging file. + Level int // Output level. + Prefix string // Prefix string for every logging content. + StSkip int // Skip count for stack. + StStatus int // Stack status(1: enabled - default; 0: disabled) + StFilter string // Stack string filter. + HeaderPrint bool `c:"header"` // Print header or not(true in default). + StdoutPrint bool `c:"stdout"` // Output to stdout or not(true in default). + LevelPrefixes map[int]string // Logging level to its prefix string mapping. + RotateSize int64 // Enables the rotate feature by set the size > 0 in bytes. + RotateBackups int // Max backups for rotated files, default is 0, means no backups. + RotateExpire time.Duration // Max expire age for rotated files. It's 0 in default, means no expiration. + RotateCompress int // Compress level for rotated files using gzip algorithm. It's 0 in default, means no compression. + RotateInterval time.Duration // Asynchronizely checks the backups and expiration at intervals. It's 1 minute in default. } // DefaultConfig returns the default configuration for logger. func DefaultConfig() Config { c := Config{ - File: gDEFAULT_FILE_FORMAT, - Flags: F_TIME_STD, - Level: LEVEL_ALL, - StStatus: 1, - HeaderPrint: true, - StdoutPrint: true, - LevelPrefixes: make(map[int]string, len(defaultLevelPrefixes)), + File: gDEFAULT_FILE_FORMAT, + Flags: F_TIME_STD, + Level: LEVEL_ALL, + StStatus: 1, + HeaderPrint: true, + StdoutPrint: true, + LevelPrefixes: make(map[int]string, len(defaultLevelPrefixes)), + RotateInterval: time.Minute, } for k, v := range defaultLevelPrefixes { c.LevelPrefixes[k] = v diff --git a/os/glog/glog_logger_rotate.go b/os/glog/glog_logger_rotate.go new file mode 100644 index 000000000..e61d8a952 --- /dev/null +++ b/os/glog/glog_logger_rotate.go @@ -0,0 +1,168 @@ +// Copyright 2020 gf Author(https://github.com/gogf/gf). All Rights Reserved. +// +// This Source Code Form is subject to the terms of the MIT License. +// If a copy of the MIT was not distributed with this file, +// You can obtain one at https://github.com/gogf/gf. + +package glog + +import ( + "fmt" + "github.com/gogf/gf/container/garray" + "github.com/gogf/gf/encoding/gcompress" + "github.com/gogf/gf/internal/intlog" + "github.com/gogf/gf/os/gfile" + "github.com/gogf/gf/os/gtime" + "github.com/gogf/gf/os/gtimer" + "github.com/gogf/gf/text/gregex" +) + +// rotateFile rotates the current logging file. +func (l *Logger) rotateFile() { + // Rotation feature is not enabled as rotation file size is zero. + if l.config.RotateSize == 0 { + return + } + l.mu.Lock() + defer l.mu.Unlock() + filePath := l.getFilePath() + // No backups, it then just removes the current logging file. + if l.config.RotateBackups == 0 { + if err := gfile.Remove(filePath); err != nil { + intlog.Print(err) + } + intlog.Printf(`%d size exceeds, no backups set, remove original logging file: %s`, l.config.RotateSize, filePath) + return + } + // Else it creates new backup files. + var ( + dirPath = gfile.Dir(filePath) + fileName = gfile.Name(filePath) + fileExt = gfile.Ext(filePath) + newFilePath = "" + ) + for { + // Rename the logging file by adding extra time information to milliseconds, like: + // access -> access.20200102190000899 + // access.log -> access.20200102190000899.log + // access.20200102.log -> access.20200102.20200102190000899.log + newFilePath = gfile.Join( + dirPath, + fmt.Sprintf(`%s.%s%s`, fileName, gtime.Now().Format("YmdHisu"), fileExt), + ) + if !gfile.Exists(newFilePath) { + break + } + } + if err := gfile.Rename(filePath, newFilePath); err != nil { + panic(err) + } +} + +// rotateChecks timely checks the backups expiration and the compression. +func (l *Logger) rotateChecks() { + defer func() { + gtimer.AddOnce(l.config.RotateInterval, l.rotateChecks) + }() + + // Checks whether file rotation not enabled. + if l.config.RotateSize == 0 || l.config.RotateBackups == 0 { + return + } + files, _ := gfile.ScanDirFile(l.config.Path, "*.*", true) + intlog.Printf("logging rotation start checks: %+v", files) + // Compression. + needCompressFileArray := garray.NewStrArray() + if l.config.RotateCompress > 0 { + for _, file := range files { + // Eg: access.20200102190000899.gz + if gfile.ExtName(file) == "gz" { + continue + } + // Eg: + // access.20200102190000899 + // access.20200102190000899.log + if gregex.IsMatchString(`.+\.\d{14,}`, file) { + needCompressFileArray.Append(file) + } + } + if needCompressFileArray.Len() > 0 { + needCompressFileArray.Iterator(func(_ int, path string) bool { + err := gcompress.GzipFile(path, path+".gz") + if err == nil { + intlog.Printf(`compressed done, remove original logging file: %s`, path) + if err = gfile.Remove(path); err != nil { + intlog.Print(err) + } + } else { + intlog.Print(err) + } + return true + }) + // Update the files array. + files, _ = gfile.ScanDirFile(l.config.Path, "*.*", true) + } + } + // Backups count limit and expiration checks. + var ( + backupFilesMap = make(map[string]*garray.SortedArray) + originalLoggingFilePath = "" + ) + if l.config.RotateBackups > 0 || l.config.RotateExpire > 0 { + for _, file := range files { + originalLoggingFilePath, _ = gregex.ReplaceString(`\.\d{14,}`, "", file) + if backupFilesMap[originalLoggingFilePath] == nil { + backupFilesMap[originalLoggingFilePath] = garray.NewSortedArray(func(a, b interface{}) int { + // Sorted by backup file mtime. + // The old backup file is put in the head of array. + file1 := a.(string) + file2 := b.(string) + result := gfile.MTimeMillisecond(file1) - gfile.MTimeMillisecond(file2) + if result <= 0 { + return -1 + } + return 1 + }) + } + if gregex.IsMatchString(`.+\.\d{14,}`, file) { + backupFilesMap[originalLoggingFilePath].Add(file) + } + } + intlog.Printf(`calculated backup files map: %+v`, backupFilesMap) + for _, array := range backupFilesMap { + for i := 0; i < array.Len()-l.config.RotateBackups; i++ { + path := array.PopLeft().(string) + intlog.Printf(`remove exceeded backup file: %s`, path) + if err := gfile.Remove(path); err != nil { + intlog.Print(err) + } + } + } + // Expiration checks. + if l.config.RotateExpire > 0 { + nowTimestampMilli := gtime.TimestampMilli() + expireMillisecond := l.config.RotateExpire.Milliseconds() + for _, array := range backupFilesMap { + array.Iterator(func(_ int, v interface{}) bool { + path := v.(string) + mtime := gfile.MTimeMillisecond(path) + differ := nowTimestampMilli - mtime + if differ > expireMillisecond { + intlog.Printf( + `%d - %d = %d > %d, remove expired backup file: %s`, + nowTimestampMilli, mtime, differ, + expireMillisecond, + path, + ) + if err := gfile.Remove(path); err != nil { + intlog.Print(err) + } + return true + } else { + return false + } + }) + } + } + } +} diff --git a/os/glog/glog_z_unit_rotate_test.go b/os/glog/glog_z_unit_rotate_test.go new file mode 100644 index 000000000..4d51ea816 --- /dev/null +++ b/os/glog/glog_z_unit_rotate_test.go @@ -0,0 +1,56 @@ +// Copyright 2020 gf Author(https://github.com/gogf/gf). All Rights Reserved. +// +// This Source Code Form is subject to the terms of the MIT License. +// If a copy of the MIT was not distributed with this file, +// You can obtain one at https://github.com/gogf/gf. + +package glog_test + +import ( + "github.com/gogf/gf/frame/g" + "github.com/gogf/gf/os/gfile" + "github.com/gogf/gf/os/glog" + "github.com/gogf/gf/os/gtime" + "github.com/gogf/gf/test/gtest" + "github.com/gogf/gf/text/gstr" + "testing" + "time" +) + +func Test_Rotate(t *testing.T) { + gtest.Case(t, func() { + l := glog.New() + p := gfile.Join(gfile.TempDir(), gtime.TimestampNanoStr()) + err := l.SetConfigWithMap(g.Map{ + "Path": p, + "File": "access.log", + "StdoutPrint": false, + "RotateSize": 10, + "RotateBackups": 2, + "RotateExpire": 5 * time.Second, + "RotateCompress": 9, + "RotateInterval": time.Second, // For unit testing only. + }) + gtest.Assert(err, nil) + defer gfile.Remove(p) + + s := "1234567890abcdefg" + for i := 0; i < 10; i++ { + l.Print(s) + } + + time.Sleep(time.Second * 3) + + files, err := gfile.ScanDirFile(p, "*.gz") + gtest.Assert(err, nil) + gtest.Assert(len(files), 2) + + content := gfile.GetContents(gfile.Join(p, "access.log")) + gtest.Assert(gstr.Count(content, s), 1) + + time.Sleep(time.Second * 4) + files, err = gfile.ScanDirFile(p, "*.gz") + gtest.Assert(err, nil) + gtest.Assert(len(files), 0) + }) +}