From a739b7d6ce7e1ab912ba8410e1586a1bbee490d7 Mon Sep 17 00:00:00 2001 From: john Date: Tue, 21 Aug 2018 21:18:56 +0800 Subject: [PATCH] =?UTF-8?q?=E5=A2=9E=E5=8A=A0gfcache=E5=8C=85=EF=BC=8C?= =?UTF-8?q?=E6=94=B9=E8=BF=9Bgcfg/gview=E5=8C=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- TODO | 3 +- g/net/ghttp/ghttp_response.go | 42 ++++------------ g/net/ghttp/ghttp_response_gzip.go | 45 +++++++++++++++++ g/net/ghttp/ghttp_response_writer.go | 43 ++++++++++++++++ g/net/ghttp/ghttp_server_config.go | 34 +++++++------ g/net/ghttp/ghttp_server_handler.go | 6 +++ g/net/ghttp/ghttp_server_log.go | 2 +- g/os/gcfg/gcfg.go | 19 +++----- g/os/gfcache/gfcache.go | 61 +++++++++++++++++++++++ g/os/gfcache/gfcache_cache.go | 73 ++++++++++++++++++++++++++++ g/os/gfile/gfile.go | 11 +++-- g/os/gfilepool/gfilepool.go | 10 ++-- g/os/gview/gview.go | 29 ++--------- geg/os/gfcache/gfcache.go | 28 +++++++++++ geg/other/test.go | 29 ++++++++--- version.go | 2 +- 16 files changed, 336 insertions(+), 101 deletions(-) create mode 100644 g/net/ghttp/ghttp_response_gzip.go create mode 100644 g/net/ghttp/ghttp_response_writer.go create mode 100644 g/os/gfcache/gfcache.go create mode 100644 g/os/gfcache/gfcache_cache.go create mode 100644 geg/os/gfcache/gfcache.go diff --git a/TODO b/TODO index b7d4d95f8..b13e0064a 100644 --- a/TODO +++ b/TODO @@ -19,6 +19,7 @@ ghttp.Server的Cookie及Session锁机制优化(去掉map锁机制); ghttp.Server增加Ip访问控制功能(DenyIps&AllowIps)； ghttp路由功能增加分组路由特性； 解决glog串日志情况； +ghttp增加返回数据压缩机制； DONE: @@ -53,4 +54,4 @@ DONE: 29. gdb增加查询缓存特性； 30. gpage分页增加对自定义后缀的支持，如:2.html, 2.php等等； 31. gvalid包增加struct tag的校验规则、自定义错误提示信息绑定的支持特性； - +32. 增加文件缓存包，可根据fsnotify机制进行缓存更新； diff --git a/g/net/ghttp/ghttp_response.go b/g/net/ghttp/ghttp_response.go index e6f7b8ae3..9d2030dec 100644 --- a/g/net/ghttp/ghttp_response.go +++ b/g/net/ghttp/ghttp_response.go @@ -8,7 +8,6 @@ package ghttp import ( - "sync" "net/http" "gitee.com/johng/gf/g/util/gconv" "gitee.com/johng/gf/g/encoding/gparser" @@ -17,46 +16,29 @@ import ( "gitee.com/johng/gf/g/frame/gins" ) -// 服务端请求返回对象 +// 服务端请求返回对象。 +// 注意该对象并没有实现http.ResponseWriter接口，而是依靠ghttp.ResponseWriter实现。 type Response struct { ResponseWriter Server *Server - Writer *ResponseWriter // io.Writer - mu sync.RWMutex // 缓冲区互斥锁 - buffer []byte // 每个请求的返回数据缓冲区 + Writer *ResponseWriter // ResponseWriter的别名 request *Request // 关联的Request请求对象 } -// 自定义的ResponseWriter，用于写入流的控制 -type ResponseWriter struct { - http.ResponseWriter - Status int // http status - Length int // response length -} - // 创建一个ghttp.Response对象指针 func newResponse(s *Server, w http.ResponseWriter) *Response { r := &Response { Server : s, - ResponseWriter : ResponseWriter{w, http.StatusOK, 0}, + ResponseWriter : ResponseWriter{ + ResponseWriter : w, + Status : http.StatusOK, + buffer : make([]byte, 0), + }, } r.Writer = &r.ResponseWriter return r } -// 覆盖父级的WriteHeader方法 -func (w *ResponseWriter) Write(buffer []byte) (int, error) { - n, e := w.ResponseWriter.Write(buffer) - w.Length += n - return n, e -} - -// 覆盖父级的WriteHeader方法 -func (w *ResponseWriter) WriteHeader(code int) { - w.Status = code - w.ResponseWriter.WriteHeader(code) -} - // 返回信息，任何变量自动转换为bytes func (r *Response) Write(content ... interface{}) { if len(content) == 0 { @@ -225,11 +207,5 @@ func (r *Response) ClearBuffer() { // 输出缓冲区数据到客户端 func (r *Response) OutputBuffer() { r.Header().Set("Server", r.Server.config.ServerAgent) - if len(r.buffer) > 0 { - r.mu.Lock() - r.ResponseWriter.Write(r.buffer) - r.buffer = make([]byte, 0) - r.mu.Unlock() - } - + r.Writer.OutputBuffer() } diff --git a/g/net/ghttp/ghttp_response_gzip.go b/g/net/ghttp/ghttp_response_gzip.go new file mode 100644 index 000000000..a557dbebf --- /dev/null +++ b/g/net/ghttp/ghttp_response_gzip.go @@ -0,0 +1,45 @@ +// Copyright 2017 gf Author(https://gitee.com/johng/gf). All Rights Reserved. +// +// This Source Code Form is subject to the terms of the MIT License. +// If a copy of the MIT was not distributed with this file, +// You can obtain one at https://gitee.com/johng/gf. +// + +package ghttp + +// 默认的gzip压缩文件类型 +var defaultGzipContentTypes = []string{ + "application/atom+xml", + "application/font-sfnt", + "application/javascript", + "application/json", + "application/ld+json", + "application/manifest+json", + "application/rdf+xml", + "application/rss+xml", + "application/schema+json", + "application/vnd.geo+json", + "application/vnd.ms-fontobject", + "application/x-font-ttf", + "application/x-javascript", + "application/x-web-app-manifest+json", + "application/xhtml+xml", + "application/xml", + "font/eot", + "font/opentype", + "image/bmp", + "image/svg+xml", + "image/vnd.microsoft.icon", + "image/x-icon", + "text/cache-manifest", + "text/css", + "text/html", + "text/javascript", + "text/plain", + "text/vcard", + "text/vnd.rim.location.xloc", + "text/vtt", + "text/x-component", + "text/x-cross-domain-policy", + "text/xml", +} \ No newline at end of file diff --git a/g/net/ghttp/ghttp_response_writer.go b/g/net/ghttp/ghttp_response_writer.go new file mode 100644 index 000000000..fbd0308bf --- /dev/null +++ b/g/net/ghttp/ghttp_response_writer.go @@ -0,0 +1,43 @@ +// Copyright 2017 gf Author(https://gitee.com/johng/gf). All Rights Reserved. +// +// This Source Code Form is subject to the terms of the MIT License. +// If a copy of the MIT was not distributed with this file, +// You can obtain one at https://gitee.com/johng/gf. +// + +package ghttp + +import ( + "net/http" + "sync" +) + +// 自定义的ResponseWriter，用于写入流的控制 +type ResponseWriter struct { + http.ResponseWriter + mu sync.RWMutex // 缓冲区互斥锁 + Status int // http status + buffer []byte // 缓冲区内容 +} + +// 覆盖父级的WriteHeader方法 +func (w *ResponseWriter) Write(buffer []byte) (int, error) { + w.buffer = append(w.buffer, buffer...) + return len(buffer), nil +} + +// 覆盖父级的WriteHeader方法 +func (w *ResponseWriter) WriteHeader(code int) { + w.Status = code + w.ResponseWriter.WriteHeader(code) +} + +// 输出buffer数据到客户端 +func (w *ResponseWriter) OutputBuffer() { + if len(w.buffer) > 0 { + w.mu.Lock() + w.ResponseWriter.Write(w.buffer) + w.buffer = make([]byte, 0) + w.mu.Unlock() + } +} diff --git a/g/net/ghttp/ghttp_server_config.go b/g/net/ghttp/ghttp_server_config.go index 167749e9a..dfe267955 100644 --- a/g/net/ghttp/ghttp_server_config.go +++ b/g/net/ghttp/ghttp_server_config.go @@ -59,27 +59,31 @@ type ServerConfig struct { AllowIps []string // 仅允许访问的ip列表，支持ip前缀过滤，如: 10 将仅允许10开头的ip访问 // 路由访问控制 DenyRoutes []string // 不允许访问的路由规则列表 + // Gzip压缩文件类型 + GzipContentTypes []string // 允许进行gzip压缩的文件类型 } // 默认HTTP Server var defaultServerConfig = ServerConfig { - Addr : "", - HTTPSAddr : "", - Handler : nil, - ReadTimeout : 60 * time.Second, - WriteTimeout : 60 * time.Second, - IdleTimeout : 60 * time.Second, - MaxHeaderBytes : 1024, - IndexFiles : []string{"index.html", "index.htm"}, - IndexFolder : false, - ServerAgent : "gf", - ServerRoot : "", + Addr : "", + HTTPSAddr : "", + Handler : nil, + ReadTimeout : 60 * time.Second, + WriteTimeout : 60 * time.Second, + IdleTimeout : 60 * time.Second, + MaxHeaderBytes : 1024, + IndexFiles : []string{"index.html", "index.htm"}, + IndexFolder : false, + ServerAgent : "gf", + ServerRoot : "", - CookieMaxAge : gDEFAULT_COOKIE_MAX_AGE, - SessionMaxAge : gDEFAULT_SESSION_MAX_AGE, - SessionIdName : gDEFAULT_SESSION_ID_NAME, + CookieMaxAge : gDEFAULT_COOKIE_MAX_AGE, + SessionMaxAge : gDEFAULT_SESSION_MAX_AGE, + SessionIdName : gDEFAULT_SESSION_ID_NAME, - ErrorLogEnabled : true, + ErrorLogEnabled : true, + + GzipContentTypes : defaultGzipContentTypes, } // 获取默认的http server设置 diff --git a/g/net/ghttp/ghttp_server_handler.go b/g/net/ghttp/ghttp_server_handler.go index 583ae2c91..bc4030747 100644 --- a/g/net/ghttp/ghttp_server_handler.go +++ b/g/net/ghttp/ghttp_server_handler.go @@ -97,6 +97,12 @@ func (s *Server)handleRequest(w http.ResponseWriter, r *http.Request) { request.Response.OutputBuffer() // 事件 - AfterOutput s.callHookHandler(HOOK_AFTER_OUTPUT, request) + + // gzip压缩处理 + encoding := request.Header.Get("Accept-Encoding") + if encoding != "" { + + } } // 初始化控制器 diff --git a/g/net/ghttp/ghttp_server_log.go b/g/net/ghttp/ghttp_server_log.go index fad5d4c47..cbc3e0287 100644 --- a/g/net/ghttp/ghttp_server_log.go +++ b/g/net/ghttp/ghttp_server_log.go @@ -26,7 +26,7 @@ func (s *Server) handleAccessLog(r *Request) { content := fmt.Sprintf(`"%s %s %s %s" %s %s`, r.Method, r.Host, r.URL.String(), r.Proto, gconv.String(r.Response.Status), - gconv.String(r.Response.Length), + gconv.String(r.Response.BufferLength()), ) content += fmt.Sprintf(` %.3f`, float64(r.LeaveTime - r.EnterTime)/1000) content += fmt.Sprintf(`, %s, "%s", "%s"`, r.GetClientIp(), r.Referer(), r.UserAgent()) diff --git a/g/os/gcfg/gcfg.go b/g/os/gcfg/gcfg.go index 90d57a0ae..9c2835ebc 100644 --- a/g/os/gcfg/gcfg.go +++ b/g/os/gcfg/gcfg.go @@ -25,7 +25,6 @@ type Config struct { name *gtype.String // 默认配置文件名称 paths *gspath.SPath // 搜索目录路径 jsons *gmap.StringInterfaceMap // 配置文件对象 - closed *gtype.Bool // 是否已经被close vc *gtype.Bool // 层级检索是否执行分隔符冲突检测(默认为false，检测会比较影响检索效率) } @@ -41,7 +40,6 @@ func New(path string, file...string) *Config { name : gtype.NewString(name), paths : s, jsons : gmap.NewStringInterfaceMap(), - closed : gtype.NewBool(), vc : gtype.NewBool(), } } @@ -251,17 +249,14 @@ func (c *Config) Reload() { c.jsons.Clear() } -// 关闭Config对象，自动关闭异步协程 -func (c *Config) Close() { - c.closed.Set(true) -} - // 添加文件监控 func (c *Config) addMonitor(path string) { - if c.jsons.Get(path) == nil { - gfsnotify.Add(path, func(event *gfsnotify.Event) { - // 删除文件内容缓存，下一次查询会自动更新 - c.jsons.Remove(event.Path) - }) + // 防止多goroutine同时调用 + if c.jsons.Get(path) != nil { + return } + gfsnotify.Add(path, func(event *gfsnotify.Event) { + // 删除文件内容缓存，下一次查询会自动更新 + c.jsons.Remove(event.Path) + }) } diff --git a/g/os/gfcache/gfcache.go b/g/os/gfcache/gfcache.go new file mode 100644 index 000000000..2e4f4eae7 --- /dev/null +++ b/g/os/gfcache/gfcache.go @@ -0,0 +1,61 @@ +// Copyright 2018 gf Author(https://gitee.com/johng/gf). All Rights Reserved. +// +// This Source Code Form is subject to the terms of the MIT License. +// If a copy of the MIT was not distributed with this file, +// You can obtain one at https://gitee.com/johng/gf. + +// 文件缓存. +package gfcache + +import ( + "gitee.com/johng/gf/g/os/gcache" + "gitee.com/johng/gf/g/container/gtype" + "gitee.com/johng/gf/g/os/gfsnotify" +) + +type Cache struct { + cap *gtype.Int // 缓存容量(byte)，设置为0表示不限制 + size *gtype.Int // 缓存大小(Byte) + cache *gcache.Cache // 缓存对象 + notify *gfsnotify.Watcher // 文件监控管理对象 +} + +const ( + // 默认的缓存容量(不限制) + gDEFAULT_CACHE_CAP = 0 +) + +var ( + // 默认的文件缓存对象 + cache = New() +) + +func New(cap ... int) *Cache { + c := gDEFAULT_CACHE_CAP + if len(cap) > 0 { + c = cap[0] + } + notify, _ := gfsnotify.New() + return &Cache { + cap : gtype.NewInt(c), + size : gtype.NewInt(), + cache : gcache.New(), + notify : notify, + } +} + + +// 获得已缓存的文件大小(byte) +func GetSize() int { + return cache.GetSize() +} + +// 获得文件内容 string +func GetContents(path string) string { + return cache.GetContents(path) +} + +// 获得文件内容 []byte +func GetBinContents(path string) []byte { + return cache.GetBinContents(path) +} \ No newline at end of file diff --git a/g/os/gfcache/gfcache_cache.go b/g/os/gfcache/gfcache_cache.go new file mode 100644 index 000000000..28cd50d99 --- /dev/null +++ b/g/os/gfcache/gfcache_cache.go @@ -0,0 +1,73 @@ +// Copyright 2018 gf Author(https://gitee.com/johng/gf). All Rights Reserved. +// +// This Source Code Form is subject to the terms of the MIT License. +// If a copy of the MIT was not distributed with this file, +// You can obtain one at https://gitee.com/johng/gf. + +// 文件缓存. +package gfcache + +import ( + "gitee.com/johng/gf/g/os/gfile" + "gitee.com/johng/gf/g/os/gfsnotify" +) + +// 设置容量大小(MB) +func (c *Cache) SetCap(cap int) { + c.cap.Set(cap) +} + +// 获得缓存容量大小(byte) +func (c *Cache) GetCap() int { + return c.cap.Val() +} + +// 获得已缓存的文件大小(byte) +func (c *Cache) GetSize() int { + return c.size.Val() +} + +// 获得文件内容 string +func (c *Cache) GetContents(path string) string { + return string(c.GetBinContents(path)) +} + +// 获得文件内容 []byte +func (c *Cache) GetBinContents(path string) []byte { + v := c.cache.Get(path) + if v != nil { + return v.([]byte) + } + b := gfile.GetBinContents(path) + if b != nil && (c.cap.Val() == 0 || c.size.Val() < c.cap.Val()) { + c.addMonitor(path) + c.cache.Set(path, b, 0) + c.size.Add(len(b)) + } + return b +} + +// 添加文件监控 +func (c *Cache) addMonitor(path string) { + // 防止多goroutine同时调用 + if c.cache.Get(path) != nil { + return + } + c.notify.Add(path, func(event *gfsnotify.Event) { + r := c.cache.Get(path).([]byte) + // 是否删除 + if event.IsRemove() { + c.cache.Remove(path) + c.size.Add(-len(r)) + } + // 更新缓存内容 + if c.cap.Val() == 0 || c.size.Val() < c.cap.Val() { + b := gfile.GetBinContents(path) + if b != nil { + dif := len(b) - len(r) + c.cache.Set(path, b, 0) + c.size.Add(dif) + } + } + }) +} \ No newline at end of file diff --git a/g/os/gfile/gfile.go b/g/os/gfile/gfile.go index 81c92dca1..9f7bd271f 100644 --- a/g/os/gfile/gfile.go +++ b/g/os/gfile/gfile.go @@ -28,16 +28,19 @@ import ( // 封装了常用的文件操作方法，如需更详细的文件控制，请查看官方os包 + // 文件分隔符 const ( Separator = string(filepath.Separator) ) -// 源码的main包所在目录，仅仅会设置一次 -var mainPkgPath = gtype.NewString() +var ( + // 源码的main包所在目录，仅仅会设置一次 + mainPkgPath = gtype.NewString() -// 编译时的 GOROOT 数值 -var goRootOfBuild = gtype.NewString() + // 编译时的 GOROOT 数值 + goRootOfBuild = gtype.NewString() +) // 给定文件的绝对路径创建文件 func Mkdir(path string) error { diff --git a/g/os/gfilepool/gfilepool.go b/g/os/gfilepool/gfilepool.go index 70801be5b..581e1a191 100644 --- a/g/os/gfilepool/gfilepool.go +++ b/g/os/gfilepool/gfilepool.go @@ -70,12 +70,14 @@ func (p *Pool) File() (*File, error) { } } -// 关闭指针池 -func (p *Pool) Close() { +// 关闭指针池(返回error是标准库io.ReadWriteCloser接口实现) +func (p *Pool) Close() error { p.pool.Close() + return nil } -// 获得底层文件指针 -func (f *File) Close() { +// 获得底层文件指针(返回error是标准库io.ReadWriteCloser接口实现) +func (f *File) Close() error { f.pool.pool.Put(f) + return nil } diff --git a/g/os/gview/gview.go b/g/os/gview/gview.go index a9d6e8a77..329b4eea7 100644 --- a/g/os/gview/gview.go +++ b/g/os/gview/gview.go @@ -12,12 +12,11 @@ import ( "bytes" "errors" "html/template" - "gitee.com/johng/gf/g/os/gfile" "gitee.com/johng/gf/g/container/gmap" "gitee.com/johng/gf/g/encoding/ghash" "gitee.com/johng/gf/g/util/gconv" - "gitee.com/johng/gf/g/os/gfsnotify" "gitee.com/johng/gf/g/os/gspath" + "gitee.com/johng/gf/g/os/gfcache" ) // 视图对象 @@ -25,7 +24,6 @@ type View struct { mu sync.RWMutex paths *gspath.SPath // 模板查找目录(绝对路径) funcmap map[string]interface{} // FuncMap - contents *gmap.StringStringMap // 已解析的模板文件内容 delimiters []string // 模板变量分隔符号 } @@ -62,7 +60,6 @@ func New(path string) *View { view := &View { paths : s, funcmap : make(map[string]interface{}), - contents : gmap.NewStringStringMap(), delimiters : make([]string, 2), } view.SetDelimiters("{{", "}}") @@ -82,18 +79,12 @@ func (view *View) AddPath(path string) error { // 解析模板，返回解析后的内容 func (view *View) Parse(file string, params...map[string]interface{}) ([]byte, error) { - path := view.paths.Search(file) - content := view.contents.Get(path) - if content == "" { - content = gfile.GetContents(path) - if content != "" { - view.addMonitor(path) - view.contents.Set(path, content) - } - } - if content == "" { + path := view.paths.Search(file) + if path == "" { return nil, errors.New("tpl \"" + file + "\" not found") } + content := gfcache.GetContents(path) + // 模板参数 data := (map[string]interface{})(nil) if len(params) > 0 { @@ -154,13 +145,3 @@ func (view *View) funcInclude(file string, data...map[string]interface{}) templa } return template.HTML(content) } - -// 添加模板文件监控 -func (view *View) addMonitor(path string) { - if view.contents.Get(path) == "" { - gfsnotify.Add(path, func(event *gfsnotify.Event) { - // 删除文件内容缓存，下一次查询会自动更新 - view.contents.Remove(event.Path) - }) - } -} diff --git a/geg/os/gfcache/gfcache.go b/geg/os/gfcache/gfcache.go new file mode 100644 index 000000000..364195847 --- /dev/null +++ b/geg/os/gfcache/gfcache.go @@ -0,0 +1,28 @@ +package main + +import ( + "gitee.com/johng/gf/g/os/gfcache" + "gitee.com/johng/gf/g/os/gfile" + "fmt" + "time" +) + +func main() { + s := 0 + r := "" + path := gfile.TempDir() + gfile.Separator + "temp" + gfile.PutContents(path, "hello") + + s = gfcache.GetSize() + r = gfcache.GetContents(path) + fmt.Println(s, r) + + gfile.PutContentsAppend(path, " john") + + // 等待1秒以便gfsnotify回调能处理完成 + time.Sleep(time.Second) + + s = gfcache.GetSize() + r = gfcache.GetContents(path) + fmt.Println(s, r) +} diff --git a/geg/other/test.go b/geg/other/test.go index c201d4e0f..dc8efcb6c 100644 --- a/geg/other/test.go +++ b/geg/other/test.go @@ -1,11 +1,28 @@ package main -import ( - "fmt" -) +import "fmt" + +type A struct { + S string +} + +type B struct { + A +} + + +func (a *A) editA () { + a.S += "a" +} + +func (b *B) editB () { + b.S += "b" +} func main() { - methodMap := make(map[string]bool) - methodMap["t"] = true - fmt.Println(methodMap["t"]) + b := new(B) + b.editA() + b.editB() + b.A.editA() + fmt.Println(b.S) } \ No newline at end of file diff --git a/version.go b/version.go index fd7ed2cd8..d952b4318 100644 --- a/version.go +++ b/version.go @@ -1,5 +1,5 @@ package gf -const VERSION = "0.99 beta" +const VERSION = "0.99.682 beta" const AUTHORS = "john"