add ClientMaxBodySize configuration for ghttp.Server

This commit is contained in:
John
2020-04-30 20:37:09 +08:00
parent b79ff84c6f
commit 45170bc53e
11 changed files with 355 additions and 73 deletions

View File

@ -9,7 +9,7 @@ import (
func Upload(r *ghttp.Request) {
saveDirPath := "/tmp/"
files := r.GetUploadFiles("upload-file")
if err := files.Save(saveDirPath); err != nil {
if _, err := files.Save(saveDirPath); err != nil {
r.Response.WriteExit(err)
}
r.Response.WriteExit("upload successfully")

View File

@ -9,7 +9,7 @@ import (
func Upload(r *ghttp.Request) {
saveDirPath := "/tmp/"
files := r.GetUploadFiles("upload-file")
if err := files.Save(saveDirPath); err != nil {
if _, err := files.Save(saveDirPath); err != nil {
r.Response.WriteExit(err)
}
r.Response.WriteExit("upload successfully")

View File

@ -1,7 +1,6 @@
package main
import (
"fmt"
"github.com/gogf/gf/frame/g"
"github.com/gogf/gf/net/ghttp"
)
@ -10,10 +9,7 @@ func main() {
s := g.Server()
s.Group("/", func(group *ghttp.RouterGroup) {
group.ALL("/", func(r *ghttp.Request) {
fmt.Println(r.GetBodyString())
fmt.Println(r.Header)
r.Response.Write(r.GetBodyString())
r.Response.Write(r.Header)
})
})
s.SetPort(8199)

View File

@ -598,7 +598,7 @@ func (s *Server) getListenerFdMap() map[string]string {
"http": "",
}
for _, v := range s.servers {
str := v.itemFunc + "#" + gconv.String(v.Fd()) + ","
str := v.address + "#" + gconv.String(v.Fd()) + ","
if v.isHttps {
if len(m["https"]) > 0 {
m["https"] += ","

View File

@ -10,6 +10,7 @@ import (
"crypto/tls"
"fmt"
"github.com/gogf/gf/internal/intlog"
"github.com/gogf/gf/util/gutil"
"net/http"
"strconv"
"time"
@ -33,55 +34,206 @@ const (
URI_TYPE_CAMEL = 3 // Method name to URI converting type, which converts name to its camel case.
)
// HTTP Server configuration.
// ServerConfig is the HTTP Server configuration manager.
type ServerConfig struct {
Address string // Basic: Server listening address like ":port", multiple addresses joined using ','.
HTTPSAddr string // Basic: HTTPS addresses, multiple addresses joined using char ','.
HTTPSCertPath string // Basic: HTTPS certification file path.
HTTPSKeyPath string // Basic: HTTPS key file path.
TLSConfig *tls.Config // Basic: TLS configuration for use by ServeTLS and ListenAndServeTLS.
Handler http.Handler // Basic: Request handler.
ReadTimeout time.Duration // Basic: Maximum duration for reading the entire request, including the body.
WriteTimeout time.Duration // Basic: Maximum duration before timing out writes of the response.
IdleTimeout time.Duration // Basic: Maximum amount of time to wait for the next request when keep-alive is enabled.
MaxHeaderBytes int // Basic: Maximum number of bytes the server will read parsing the request header's keys and values, including the request line.
KeepAlive bool // Basic: Enable HTTP keep-alive.
ServerAgent string // Basic: Server agent information.
View *gview.View // Basic: View object for the server.
Rewrites map[string]string // Static: URI rewrite rules map.
IndexFiles []string // Static: The index files for static folder.
IndexFolder bool // Static: List sub-files when requesting folder; server responses HTTP status code 403 if false.
ServerRoot string // Static: The root directory for static service.
SearchPaths []string // Static: Additional searching directories for static service.
StaticPaths []staticPathItem // Static: URI to directory mapping array.
FileServerEnabled bool // Static: Switch for static service.
CookieMaxAge time.Duration // Cookie: Max TTL for cookie items.
CookiePath string // Cookie: Cookie Path(also affects the default storage for session id).
CookieDomain string // Cookie: Cookie Domain(also affects the default storage for session id).
SessionMaxAge time.Duration // Session: Max TTL for session items.
SessionIdName string // Session: Session id name.
SessionPath string // Session: Session Storage directory path for storing session files.
SessionStorage gsession.Storage // Session: Session Storage implementer.
Logger *glog.Logger // Logging: Logger for server.
LogPath string // Logging: Directory for storing logging files.
LogStdout bool // Logging: Printing logging content to stdout.
ErrorStack bool // Logging: Logging stack information when error.
ErrorLogEnabled bool // Logging: Enable error logging files.
ErrorLogPattern string // Logging: Error log file pattern like: error-{Ymd}.log
AccessLogEnabled bool // Logging: Enable access logging files.
AccessLogPattern string // Logging: Error log file pattern like: access-{Ymd}.log
PProfEnabled bool // PProf: Enable PProf feature.
PProfPattern string // PProf: PProf service pattern for router.
FormParsingMemory int64 // Other: Max memory in bytes which can be used for parsing multimedia form.
NameToUriType int // Other: Type for converting struct method name to URI when registering routes.
RouteOverWrite bool // Other: Allow overwrite the route if duplicated.
DumpRouterMap bool // Other: Whether automatically dump router map when server starts.
Graceful bool // Other: Enable graceful reload feature for all servers of the process.
// ==================================
// Basic.
// ==================================
// Address specifies the server listening address like "port" or ":port",
// multiple addresses joined using ','.
Address string
// HTTPSAddr specifies the HTTPS addresses, multiple addresses joined using char ','.
HTTPSAddr string
// HTTPSCertPath specifies certification file path for HTTPS service.
HTTPSCertPath string
// HTTPSKeyPath specifies the key file path for HTTPS service.
HTTPSKeyPath string
// TLSConfig optionally provides a TLS configuration for use
// by ServeTLS and ListenAndServeTLS. Note that this value is
// cloned by ServeTLS and ListenAndServeTLS, so it's not
// possible to modify the configuration with methods like
// tls.Config.SetSessionTicketKeys. To use
// SetSessionTicketKeys, use Server.Serve with a TLS Listener
// instead.
TLSConfig *tls.Config
// Handler the handler for HTTP request.
Handler http.Handler
// ReadTimeout is the maximum duration for reading the entire
// request, including the body.
//
// Because ReadTimeout does not let Handlers make per-request
// decisions on each request body's acceptable deadline or
// upload rate, most users will prefer to use
// ReadHeaderTimeout. It is valid to use them both.
ReadTimeout time.Duration
// WriteTimeout is the maximum duration before timing out
// writes of the response. It is reset whenever a new
// request's header is read. Like ReadTimeout, it does not
// let Handlers make decisions on a per-request basis.
WriteTimeout time.Duration
// IdleTimeout is the maximum amount of time to wait for the
// next request when keep-alives are enabled. If IdleTimeout
// is zero, the value of ReadTimeout is used. If both are
// zero, there is no timeout.
IdleTimeout time.Duration
// MaxHeaderBytes controls the maximum number of bytes the
// server will read parsing the request header's keys and
// values, including the request line. It does not limit the
// size of the request body.
//
// It can be configured in configuration file using string like: 1m, 10m, 500kb etc.
// It's 1024 bytes in default.
MaxHeaderBytes int
// KeepAlive enables HTTP keep-alive.
KeepAlive bool
// ServerAgent specifies the server agent information, which is wrote to
// HTTP response header as "Server".
ServerAgent string
// View specifies the default template view object for the server.
View *gview.View
// ==================================
// Static.
// ==================================
// Rewrites specifies the URI rewrite rules map.
Rewrites map[string]string
// IndexFiles specifies the index files for static folder.
IndexFiles []string
// IndexFolder specifies if listing sub-files when requesting folder.
// The server responses HTTP status code 403 if it is false.
IndexFolder bool
// ServerRoot specifies the root directory for static service.
ServerRoot string
// SearchPaths specifies additional searching directories for static service.
SearchPaths []string
// StaticPaths specifies URI to directory mapping array.
StaticPaths []staticPathItem
// FileServerEnabled is the global switch for static service.
// It is automatically set enabled if any static path is set.
FileServerEnabled bool
// ==================================
// Cookie.
// ==================================
// CookieMaxAge specifies the max TTL for cookie items.
CookieMaxAge time.Duration
// CookiePath specifies cookie path.
// It also affects the default storage for session id.
CookiePath string
// CookieDomain specifies cookie domain.
// It also affects the default storage for session id.
CookieDomain string
// ==================================
// Session.
// ==================================
// SessionMaxAge specifies max TTL for session items.
SessionMaxAge time.Duration
// SessionIdName specifies the session id name.
SessionIdName string
// SessionPath specifies the session storage directory path for storing session files.
// It only makes sense if the session storage is type of file storage.
SessionPath string
// SessionStorage specifies the session storage.
SessionStorage gsession.Storage
// ==================================
// Logging.
// ==================================
// Logger specifies the logger for server.
Logger *glog.Logger
// LogPath specifies the directory for storing logging files.
LogPath string
// LogStdout specifies whether printing logging content to stdout.
LogStdout bool
// ErrorStack specifies whether logging stack information when error.
ErrorStack bool
// ErrorLogEnabled enables error logging content to files.
ErrorLogEnabled bool
// ErrorLogPattern specifies the error log file pattern like: error-{Ymd}.log
ErrorLogPattern string
// AccessLogEnabled enables access logging content to files.
AccessLogEnabled bool
// AccessLogPattern specifies the error log file pattern like: access-{Ymd}.log
AccessLogPattern string
// ==================================
// PProf.
// ==================================
// PProfEnabled enables PProf feature.
PProfEnabled bool
// PProfPattern specifies the PProf service pattern for router.
PProfPattern string
// ==================================
// Other.
// ==================================
// ClientMaxBodySize specifies the max body size limit in bytes for client request.
// It can be configured in configuration file using string like: 1m, 10m, 500kb etc.
// It's 8MB in default.
ClientMaxBodySize int64
// FormParsingMemory specifies max memory buffer size in bytes which can be used for
// parsing multimedia form.
// It can be configured in configuration file using string like: 1m, 10m, 500kb etc.
// It's 1MB in default.
FormParsingMemory int64
// NameToUriType specifies the type for converting struct method name to URI when
// registering routes.
NameToUriType int
// RouteOverWrite allows overwrite the route if duplicated.
RouteOverWrite bool
// DumpRouterMap specifies whether automatically dumps router map when server starts.
DumpRouterMap bool
// Graceful enables graceful reload feature for all servers of the process.
Graceful bool
}
// Config returns the default ServerConfig object.
// Note that, do not define this default configuration to local variable,
// as there're some pointer attributes that may be shared in different servers.
// Config creates and returns a ServerConfig object with default configurations.
// Note that, do not define this default configuration to local package variable, as there're
// some pointer attributes that may be shared in different servers.
func Config() ServerConfig {
return ServerConfig{
Address: "",
@ -112,7 +264,8 @@ func Config() ServerConfig {
AccessLogEnabled: false,
AccessLogPattern: "access-{Ymd}.log",
DumpRouterMap: true,
FormParsingMemory: 100 * 1024 * 1024, // 100MB
ClientMaxBodySize: 8 * 1024 * 1024, // 8MB
FormParsingMemory: 1024 * 1024, // 1MB
Rewrites: make(map[string]string),
Graceful: true,
}
@ -130,6 +283,21 @@ func ConfigFromMap(m map[string]interface{}) (ServerConfig, error) {
// SetConfigWithMap sets the configuration for the server using map.
func (s *Server) SetConfigWithMap(m map[string]interface{}) error {
// The m now is a shallow copy of m.
// Any changes to m does not affect the original one.
// A little tricky, isn't it?
m = gutil.MapCopy(m)
// Allow setting the size configuration items using string size like:
// 1m, 100mb, 512kb, etc.
if k, v := gutil.MapPossibleItemByKey(m, "MaxHeaderBytes"); k != "" {
m[k] = gfile.StrToSize(gconv.String(v))
}
if k, v := gutil.MapPossibleItemByKey(m, "ClientMaxBodySize"); k != "" {
m[k] = gfile.StrToSize(gconv.String(v))
}
if k, v := gutil.MapPossibleItemByKey(m, "FormParsingMemory"); k != "" {
m[k] = gfile.StrToSize(gconv.String(v))
}
// Update the current configuration object.
if err := gconv.Struct(m, &s.config); err != nil {
return err
@ -156,7 +324,6 @@ func (s *Server) SetConfig(c ServerConfig) error {
s.EnableHTTPS(c.HTTPSCertPath, c.HTTPSKeyPath)
}
SetGraceful(c.Graceful)
intlog.Printf("SetConfig: %+v", s.config)
return nil
}
@ -168,7 +335,7 @@ func (s *Server) SetAddr(address string) {
}
// SetPort sets the listening ports for the server.
// The listening ports can be multiple.
// The listening ports can be multiple like: SetPort(80, 8080).
func (s *Server) SetPort(port ...int) {
if len(port) > 0 {
s.config.Address = ""
@ -187,7 +354,7 @@ func (s *Server) SetHTTPSAddr(address string) {
}
// SetHTTPSPort sets the HTTPS listening ports for the server.
// The listening ports can be multiple.
// The listening ports can be multiple like: SetHTTPSPort(443, 500).
func (s *Server) SetHTTPSPort(port ...int) {
if len(port) > 0 {
s.config.HTTPSAddr = ""

View File

@ -14,6 +14,10 @@ func (s *Server) SetDumpRouterMap(enabled bool) {
s.config.DumpRouterMap = enabled
}
func (s *Server) SetClientMaxBodySize(maxSize int64) {
s.config.ClientMaxBodySize = maxSize
}
func (s *Server) SetFormParsingMemory(maxMemory int64) {
s.config.FormParsingMemory = maxMemory
}

View File

@ -12,6 +12,7 @@ import (
"errors"
"fmt"
"github.com/gogf/gf/os/gproc"
"github.com/gogf/gf/text/gstr"
"log"
"net"
"net/http"
@ -22,7 +23,7 @@ import (
type gracefulServer struct {
server *Server // Belonged server.
fd uintptr // 热重启时传递的socket监听文件句柄
itemFunc string // 监听地址信息
address string // 监听地址信息
httpServer *http.Server // 底层http.Server
rawListener net.Listener // 原始listener
listener net.Listener // 接口化封装的listener
@ -31,11 +32,15 @@ type gracefulServer struct {
}
// 创建一个优雅的Http Server
func (s *Server) newGracefulServer(itemFunc string, fd ...int) *gracefulServer {
func (s *Server) newGracefulServer(address string, fd ...int) *gracefulServer {
// Change port to address like: 80 -> :80
if gstr.IsNumeric(address) {
address = ":" + address
}
gs := &gracefulServer{
server: s,
itemFunc: itemFunc,
httpServer: s.newHttpServer(itemFunc),
address: address,
httpServer: s.newHttpServer(address),
}
// 是否有继承的文件描述符
if len(fd) > 0 && fd[0] > 0 {
@ -45,9 +50,9 @@ func (s *Server) newGracefulServer(itemFunc string, fd ...int) *gracefulServer {
}
// 生成一个底层的Web Server对象
func (s *Server) newHttpServer(itemFunc string) *http.Server {
func (s *Server) newHttpServer(address string) *http.Server {
server := &http.Server{
Addr: itemFunc,
Addr: address,
Handler: s.config.Handler,
ReadTimeout: s.config.ReadTimeout,
WriteTimeout: s.config.WriteTimeout,
@ -61,8 +66,7 @@ func (s *Server) newHttpServer(itemFunc string) *http.Server {
// 执行HTTP监听
func (s *gracefulServer) ListenAndServe() error {
itemFunc := s.httpServer.Addr
ln, err := s.getNetListener(itemFunc)
ln, err := s.getNetListener()
if err != nil {
return err
}
@ -89,7 +93,6 @@ func (s *gracefulServer) setFd(fd int) {
// 执行HTTPS监听
func (s *gracefulServer) ListenAndServeTLS(certFile, keyFile string, tlsConfig ...*tls.Config) error {
itemFunc := s.httpServer.Addr
var config *tls.Config
if len(tlsConfig) > 0 && tlsConfig[0] != nil {
config = tlsConfig[0]
@ -109,7 +112,7 @@ func (s *gracefulServer) ListenAndServeTLS(certFile, keyFile string, tlsConfig .
if err != nil {
return errors.New(fmt.Sprintf(`open cert file "%s","%s" failed: %s`, certFile, keyFile, err.Error()))
}
ln, err := s.getNetListener(itemFunc)
ln, err := s.getNetListener()
if err != nil {
return err
}
@ -134,7 +137,10 @@ func (s *gracefulServer) doServe() error {
if s.fd != 0 {
action = "reloaded"
}
s.server.Logger().Printf("%d: %s server %s listening on [%s]", gproc.Pid(), s.getProto(), action, s.itemFunc)
s.server.Logger().Printf(
"%d: %s server %s listening on [%s]",
gproc.Pid(), s.getProto(), action, s.address,
)
s.status = SERVER_STATUS_RUNNING
err := s.httpServer.Serve(s.listener)
s.status = SERVER_STATUS_STOPPED
@ -142,7 +148,7 @@ func (s *gracefulServer) doServe() error {
}
// 自定义的net.Listener
func (s *gracefulServer) getNetListener(itemFunc string) (net.Listener, error) {
func (s *gracefulServer) getNetListener() (net.Listener, error) {
var ln net.Listener
var err error
if s.fd > 0 {
@ -153,7 +159,7 @@ func (s *gracefulServer) getNetListener(itemFunc string) (net.Listener, error) {
return nil, err
}
} else {
ln, err = net.Listen("tcp", itemFunc)
ln, err = net.Listen("tcp", s.httpServer.Addr)
if err != nil {
err = fmt.Errorf("%d: net.Listen error: %v", gproc.Pid(), err)
}
@ -167,7 +173,10 @@ func (s *gracefulServer) shutdown() {
return
}
if err := s.httpServer.Shutdown(context.Background()); err != nil {
s.server.Logger().Errorf("%d: %s server [%s] shutdown error: %v", gproc.Pid(), s.getProto(), s.itemFunc, err)
s.server.Logger().Errorf(
"%d: %s server [%s] shutdown error: %v",
gproc.Pid(), s.getProto(), s.address, err,
)
}
}
@ -177,6 +186,9 @@ func (s *gracefulServer) close() {
return
}
if err := s.httpServer.Close(); err != nil {
s.server.Logger().Errorf("%d: %s server [%s] closed error: %v", gproc.Pid(), s.getProto(), s.itemFunc, err)
s.server.Logger().Errorf(
"%d: %s server [%s] closed error: %v",
gproc.Pid(), s.getProto(), s.address, err,
)
}
}

View File

@ -26,6 +26,10 @@ import (
// 默认HTTP Server处理入口http包底层默认使用了gorutine异步处理请求所以这里不再异步执行
func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
// Max body size limit.
if s.config.ClientMaxBodySize > 0 {
r.Body = http.MaxBytesReader(w, r.Body, s.config.ClientMaxBodySize)
}
// 重写规则判断
if len(s.config.Rewrites) > 0 {
if rewrite, ok := s.config.Rewrites[r.URL.Path]; ok {

View File

@ -7,6 +7,10 @@
package ghttp_test
import (
"fmt"
"github.com/gogf/gf/os/gfile"
"github.com/gogf/gf/os/gtime"
"github.com/gogf/gf/text/gstr"
"testing"
"time"
@ -58,3 +62,97 @@ func Test_SetConfigWithMap(t *testing.T) {
t.Assert(err, nil)
})
}
func Test_ClientMaxBodySize(t *testing.T) {
p, _ := ports.PopRand()
s := g.Server(p)
s.Group("/", func(group *ghttp.RouterGroup) {
group.POST("/", func(r *ghttp.Request) {
r.Response.Write(r.GetBodyString())
})
})
m := g.Map{
"Address": p,
"ClientMaxBodySize": "1k",
}
gtest.Assert(s.SetConfigWithMap(m), nil)
s.SetPort(p)
s.SetDumpRouterMap(false)
s.Start()
defer s.Shutdown()
time.Sleep(100 * time.Millisecond)
gtest.C(t, func(t *gtest.T) {
c := g.Client()
c.SetPrefix(fmt.Sprintf("http://127.0.0.1:%d", p))
data := make([]byte, 1056)
for i := 0; i < 1056; i++ {
data[i] = 'a'
}
t.Assert(
gstr.Trim(c.PostContent("/", data)),
data[:1024],
)
})
}
func Test_ClientMaxBodySize_File(t *testing.T) {
p, _ := ports.PopRand()
s := g.Server(p)
s.Group("/", func(group *ghttp.RouterGroup) {
group.POST("/", func(r *ghttp.Request) {
r.GetUploadFile("file")
r.Response.Write("ok")
})
})
m := g.Map{
"Address": p,
"ErrorLogEnabled": false,
"ClientMaxBodySize": "1k",
}
gtest.Assert(s.SetConfigWithMap(m), nil)
s.SetPort(p)
s.SetDumpRouterMap(false)
s.Start()
defer s.Shutdown()
time.Sleep(100 * time.Millisecond)
// ok
gtest.C(t, func(t *gtest.T) {
c := g.Client()
c.SetPrefix(fmt.Sprintf("http://127.0.0.1:%d", p))
path := gfile.TempDir(gtime.TimestampNanoStr())
data := make([]byte, 512)
for i := 0; i < 512; i++ {
data[i] = 'a'
}
t.Assert(gfile.PutBytes(path, data), nil)
defer gfile.Remove(path)
t.Assert(
gstr.Trim(c.PostContent("/", "name=john&file=@file:"+path)),
"ok",
)
})
// too large
gtest.C(t, func(t *gtest.T) {
c := g.Client()
c.SetPrefix(fmt.Sprintf("http://127.0.0.1:%d", p))
path := gfile.TempDir(gtime.TimestampNanoStr())
data := make([]byte, 1056)
for i := 0; i < 1056; i++ {
data[i] = 'a'
}
t.Assert(gfile.PutBytes(path, data), nil)
defer gfile.Remove(path)
t.Assert(
gstr.Trim(c.PostContent("/", "name=john&file=@file:"+path)),
"http: request body too large",
)
})
}

View File

@ -27,7 +27,7 @@ func ReadableSize(path string) string {
return FormatSize(Size(path))
}
// StrToSize converts formatted size string to its size in bytes from Bytes to BrontoByte.
// StrToSize converts formatted size string to its size in bytes.
func StrToSize(sizeStr string) int64 {
i := 0
for ; i < len(sizeStr); i++ {

View File

@ -77,6 +77,7 @@ func (view *View) SetConfigWithMap(m map[string]interface{}) error {
return errors.New("configuration cannot be empty")
}
// The m now is a shallow copy of m.
// Any changes to m does not affect the original one.
// A little tricky, isn't it?
m = gutil.MapCopy(m)
// Most common used configuration support for single view path.