From ba56eb87b1a164b9f33f86d2dd671691dfa0036c Mon Sep 17 00:00:00 2001 From: John Date: Thu, 26 Mar 2020 20:58:57 +0800 Subject: [PATCH] improve rotation feature for package glog --- .example/other/test.go | 20 ++++------ os/gfile/gfile.go | 3 +- os/glog/glog_logger.go | 6 +-- os/glog/glog_logger_config.go | 52 +++++++++++++------------- os/glog/glog_logger_rotate.go | 60 ++++++++++++++++++------------ os/glog/glog_z_unit_rotate_test.go | 32 ++++++++-------- 6 files changed, 91 insertions(+), 82 deletions(-) diff --git a/.example/other/test.go b/.example/other/test.go index 76b87e09c..f72598ed7 100644 --- a/.example/other/test.go +++ b/.example/other/test.go @@ -1,20 +1,14 @@ package main import ( - "os" - "time" + "fmt" + "github.com/gogf/gf/os/gfile" + "github.com/gogf/gf/text/gregex" ) func main() { - file, err := os.Create("/tmp/testfile") - if err != nil { - panic(err) - } - for { - _, err = file.Write([]byte("test\n")) - if err != nil { - panic(err) - } - time.Sleep(5 * time.Second) - } + fmt.Println(gfile.Basename("/tmp/1585227151172826000/access.20200326205231173924.log")) + fmt.Println( + gregex.IsMatchString(`.+\.\d{20}\.log`, + gfile.Basename("/tmp/1585227151172826000/access.20200326205231173924.log"))) } diff --git a/os/gfile/gfile.go b/os/gfile/gfile.go index f230bbad0..54dbbc9be 100644 --- a/os/gfile/gfile.go +++ b/os/gfile/gfile.go @@ -177,11 +177,12 @@ func Stat(path string) (os.FileInfo, error) { } // Move renames (moves) to path. +// If already exists and is not a directory, it'll be replaced. func Move(src string, dst string) error { return os.Rename(src, dst) } -// Alias of Move. +// Rename is alias of Move. // See Move. func Rename(src string, dst string) error { return Move(src, dst) diff --git a/os/glog/glog_logger.go b/os/glog/glog_logger.go index 6c0eef341..c2632ee67 100644 --- a/os/glog/glog_logger.go +++ b/os/glog/glog_logger.go @@ -57,9 +57,9 @@ func New() *Logger { logger := &Logger{ config: DefaultConfig(), } - // Initialize the internal handler after one second. - gtimer.AddOnce(time.Second, func() { - gtimer.AddOnce(logger.config.RotateInterval, logger.rotateChecksTimely) + // Initialize the internal handler after some delay. + gtimer.AddOnce(500*time.Millisecond, func() { + gtimer.AddOnce(logger.config.RotateCheckInterval, logger.rotateChecksTimely) }) return logger } diff --git a/os/glog/glog_logger_config.go b/os/glog/glog_logger_config.go index e406bb73e..fee886e87 100644 --- a/os/glog/glog_logger_config.go +++ b/os/glog/glog_logger_config.go @@ -20,37 +20,37 @@ import ( // Config is the configuration object for logger. type Config struct { - Writer io.Writer // Customized io.Writer. - Flags int // Extra flags for logging output features. - Path string // Logging directory path. - File string // Format for logging file. - Level int // Output level. - Prefix string // Prefix string for every logging content. - StSkip int // Skip count for stack. - StStatus int // Stack status(1: enabled - default; 0: disabled) - StFilter string // Stack string filter. - HeaderPrint bool `c:"header"` // Print header or not(true in default). - StdoutPrint bool `c:"stdout"` // Output to stdout or not(true in default). - LevelPrefixes map[int]string // Logging level to its prefix string mapping. - RotateSize int64 // Rotate the logging file if its size > 0 in bytes. - RotateExpire time.Duration // Rotate the logging file if its mtime exceeds this duration. - RotateBackLimit int // Max backup for rotated files, default is 0, means no backups. - RotateBackExpire time.Duration // Max expire for rotated files, which is 0 in default, means no expiration. - RotateBackCompress int // Compress level for rotated files using gzip algorithm. It's 0 in default, means no compression. - RotateInterval time.Duration // Asynchronizely checks the backups and expiration at intervals. It's 1 hour in default. + Writer io.Writer // Customized io.Writer. + Flags int // Extra flags for logging output features. + Path string // Logging directory path. + File string // Format for logging file. + Level int // Output level. + Prefix string // Prefix string for every logging content. + StSkip int // Skip count for stack. + StStatus int // Stack status(1: enabled - default; 0: disabled) + StFilter string // Stack string filter. + HeaderPrint bool `c:"header"` // Print header or not(true in default). + StdoutPrint bool `c:"stdout"` // Output to stdout or not(true in default). + LevelPrefixes map[int]string // Logging level to its prefix string mapping. + RotateSize int64 // Rotate the logging file if its size > 0 in bytes. + RotateExpire time.Duration // Rotate the logging file if its mtime exceeds this duration. + RotateBackupLimit int // Max backup for rotated files, default is 0, means no backups. + RotateBackupExpire time.Duration // Max expire for rotated files, which is 0 in default, means no expiration. + RotateBackupCompress int // Compress level for rotated files using gzip algorithm. It's 0 in default, means no compression. + RotateCheckInterval time.Duration // Asynchronizely checks the backups and expiration at intervals. It's 1 hour in default. } // DefaultConfig returns the default configuration for logger. func DefaultConfig() Config { c := Config{ - File: gDEFAULT_FILE_FORMAT, - Flags: F_TIME_STD, - Level: LEVEL_ALL, - StStatus: 1, - HeaderPrint: true, - StdoutPrint: true, - LevelPrefixes: make(map[int]string, len(defaultLevelPrefixes)), - RotateInterval: time.Hour, + File: gDEFAULT_FILE_FORMAT, + Flags: F_TIME_STD, + Level: LEVEL_ALL, + StStatus: 1, + HeaderPrint: true, + StdoutPrint: true, + LevelPrefixes: make(map[int]string, len(defaultLevelPrefixes)), + RotateCheckInterval: time.Hour, } for k, v := range defaultLevelPrefixes { c.LevelPrefixes[k] = v diff --git a/os/glog/glog_logger_rotate.go b/os/glog/glog_logger_rotate.go index 9e2f3caa4..54185cffa 100644 --- a/os/glog/glog_logger_rotate.go +++ b/os/glog/glog_logger_rotate.go @@ -34,7 +34,7 @@ func (l *Logger) rotateFileBySize(now time.Time) { // doRotateFile rotates the given logging file. func (l *Logger) doRotateFile(filePath string) error { // No backups, it then just removes the current logging file. - if l.config.RotateBackLimit == 0 { + if l.config.RotateBackupLimit == 0 { if err := gfile.Remove(filePath); err != nil { return err } @@ -45,16 +45,31 @@ func (l *Logger) doRotateFile(filePath string) error { var ( dirPath = gfile.Dir(filePath) fileName = gfile.Name(filePath) - fileExt = gfile.Ext(filePath) + fileExtName = gfile.ExtName(filePath) newFilePath = "" ) - // Rename the logging file by adding extra time information to milliseconds, like: - // access.log -> access.20200102190000899.log - // access.20200102.log -> access.20200102.20200102190000899.log - newFilePath = gfile.Join( - dirPath, - fmt.Sprintf(`%s.%s%s`, fileName, gtime.Now().Format("YmdHisu"), fileExt), - ) + // Rename the logging file by adding extra datetime information to microseconds, like: + // access.log -> access.20200326101301899002.log + // access.20200102.log -> access.20200102.20200326101301899002.log + for { + var ( + now = gtime.Now() + micro = now.Microsecond() % 1000 + ) + for micro < 100 { + micro *= 10 + } + newFilePath = gfile.Join( + dirPath, + fmt.Sprintf( + `%s.%s%d.%s`, + fileName, now.Format("YmdHisu"), micro, fileExtName, + ), + ) + if !gfile.Exists(newFilePath) { + break + } + } if err := gfile.Rename(filePath, newFilePath); err != nil { return err } @@ -63,7 +78,7 @@ func (l *Logger) doRotateFile(filePath string) error { // rotateChecksTimely timely checks the backups expiration and the compression. func (l *Logger) rotateChecksTimely() { - defer gtimer.AddOnce(l.config.RotateInterval, l.rotateChecksTimely) + defer gtimer.AddOnce(l.config.RotateCheckInterval, l.rotateChecksTimely) // Checks whether file rotation not enabled. if l.config.RotateSize <= 0 && l.config.RotateExpire == 0 { return @@ -110,16 +125,15 @@ func (l *Logger) rotateChecksTimely() { // Rotated file compression. // ============================================================= needCompressFileArray := garray.NewStrArray() - if l.config.RotateBackCompress > 0 { + if l.config.RotateBackupCompress > 0 { for _, file := range files { - // Eg: access.20200102190000899.log.gz + // Eg: access.20200326101301899002.log.gz if gfile.ExtName(file) == "gz" { continue } // Eg: - // access.20200102190000899 - // access.20200102190000899.log - if gregex.IsMatchString(`.+\.\d{14,}`, gfile.Name(file)) { + // access.20200326101301899002.log + if gregex.IsMatchString(`.+\.\d{20}\.log`, gfile.Basename(file)) { needCompressFileArray.Append(file) } } @@ -148,9 +162,9 @@ func (l *Logger) rotateChecksTimely() { backupFilesMap = make(map[string]*garray.SortedArray) originalLoggingFilePath = "" ) - if l.config.RotateBackLimit > 0 || l.config.RotateBackExpire > 0 { + if l.config.RotateBackupLimit > 0 || l.config.RotateBackupExpire > 0 { for _, file := range files { - originalLoggingFilePath, _ = gregex.ReplaceString(`\.\d{14,}`, "", file) + originalLoggingFilePath, _ = gregex.ReplaceString(`\.\d{20}`, "", file) if backupFilesMap[originalLoggingFilePath] == nil { backupFilesMap[originalLoggingFilePath] = garray.NewSortedArray(func(a, b interface{}) int { // Sorted by rotated/backup file mtime. @@ -165,23 +179,23 @@ func (l *Logger) rotateChecksTimely() { }) } // Check if this file a rotated/backup file. - if gregex.IsMatchString(`.+\.\d{14,}`, gfile.Name(file)) { + if gregex.IsMatchString(`.+\.\d{20}\.log`, gfile.Basename(file)) { backupFilesMap[originalLoggingFilePath].Add(file) } } intlog.Printf(`calculated backup files map: %+v`, backupFilesMap) for _, array := range backupFilesMap { - diff := array.Len() - l.config.RotateBackLimit + diff := array.Len() - l.config.RotateBackupLimit for i := 0; i < diff; i++ { path := array.PopLeft().(string) - intlog.Printf(`remove exceeded backup file: %s`, path) + intlog.Printf(`remove exceeded backup limit file: %s`, path) if err := gfile.Remove(path); err != nil { intlog.Print(err) } } } // Backup expiration checks. - if l.config.RotateBackExpire > 0 { + if l.config.RotateBackupExpire > 0 { var ( mtime time.Time subDuration time.Duration @@ -191,10 +205,10 @@ func (l *Logger) rotateChecksTimely() { path := v.(string) mtime = gfile.MTime(path) subDuration = now.Sub(mtime) - if subDuration > l.config.RotateBackExpire { + if subDuration > l.config.RotateBackupExpire { intlog.Printf( `%v - %v = %v > %v, remove expired backup file: %s`, - now, mtime, subDuration, l.config.RotateBackExpire, path, + now, mtime, subDuration, l.config.RotateBackupExpire, path, ) if err := gfile.Remove(path); err != nil { intlog.Print(err) diff --git a/os/glog/glog_z_unit_rotate_test.go b/os/glog/glog_z_unit_rotate_test.go index efba93398..941af4280 100644 --- a/os/glog/glog_z_unit_rotate_test.go +++ b/os/glog/glog_z_unit_rotate_test.go @@ -22,14 +22,14 @@ func Test_Rotate_Size(t *testing.T) { l := glog.New() p := gfile.Join(gfile.TempDir(), gtime.TimestampNanoStr()) err := l.SetConfigWithMap(g.Map{ - "Path": p, - "File": "access.log", - "StdoutPrint": false, - "RotateSize": 10, - "RotateBackLimit": 2, - "RotateBackExpire": 5 * time.Second, - "RotateBackCompress": 9, - "RotateInterval": time.Second, // For unit testing only. + "Path": p, + "File": "access.log", + "StdoutPrint": false, + "RotateSize": 10, + "RotateBackupLimit": 2, + "RotateBackupExpire": 5 * time.Second, + "RotateBackupCompress": 9, + "RotateCheckInterval": time.Second, // For unit testing only. }) t.Assert(err, nil) defer gfile.Remove(p) @@ -60,14 +60,14 @@ func Test_Rotate_Expire(t *testing.T) { l := glog.New() p := gfile.Join(gfile.TempDir(), gtime.TimestampNanoStr()) err := l.SetConfigWithMap(g.Map{ - "Path": p, - "File": "access.log", - "StdoutPrint": false, - "RotateExpire": time.Second, - "RotateBackLimit": 2, - "RotateBackExpire": 5 * time.Second, - "RotateBackCompress": 9, - "RotateInterval": time.Second, // For unit testing only. + "Path": p, + "File": "access.log", + "StdoutPrint": false, + "RotateExpire": time.Second, + "RotateBackupLimit": 2, + "RotateBackupExpire": 5 * time.Second, + "RotateBackupCompress": 9, + "RotateCheckInterval": time.Second, // For unit testing only. }) t.Assert(err, nil) defer gfile.Remove(p)