diff --git a/database/gdb/gdb.go b/database/gdb/gdb.go index 10672ca3a..6a3772397 100644 --- a/database/gdb/gdb.go +++ b/database/gdb/gdb.go @@ -35,6 +35,7 @@ type DB interface { // The DB interface is designed not only for // relational databases but also for NoSQL databases in the future. The name // "Table" is not proper for that purpose any more. + // Also see Core.Table. // Deprecated, use Model instead. Table(table ...string) *Model @@ -46,144 +47,151 @@ type DB interface { // Model("user, user_detail") // Model("user u, user_detail ud") // 2. Model name with alias: Model("user", "u") + // Also see Core.Model. Model(table ...string) *Model // Schema creates and returns a schema. + // Also see Core.Schema. Schema(schema string) *Schema // With creates and returns an ORM model based on meta data of given object. + // Also see Core.With. With(object interface{}) *Model // Open creates a raw connection object for database with given node configuration. // Note that it is not recommended using the this function manually. + // Also see DriverMysql.Open. Open(config *ConfigNode) (*sql.DB, error) // Ctx is a chaining function, which creates and returns a new DB that is a shallow copy // of current DB object and with given context in it. // Note that this returned DB object can be used only once, so do not assign it to // a global or package variable for long using. + // Also see Core.Ctx. Ctx(ctx context.Context) DB // =========================================================================== // Query APIs. // =========================================================================== - Query(sql string, args ...interface{}) (*sql.Rows, error) - Exec(sql string, args ...interface{}) (sql.Result, error) - Prepare(sql string, execOnMaster ...bool) (*Stmt, error) + Query(sql string, args ...interface{}) (*sql.Rows, error) // See Core.Query. + Exec(sql string, args ...interface{}) (sql.Result, error) // See Core.Exec. + Prepare(sql string, execOnMaster ...bool) (*Stmt, error) // See Core.Prepare. // =========================================================================== // Common APIs for CURD. // =========================================================================== - Insert(table string, data interface{}, batch ...int) (sql.Result, error) - InsertIgnore(table string, data interface{}, batch ...int) (sql.Result, error) - Replace(table string, data interface{}, batch ...int) (sql.Result, error) - Save(table string, data interface{}, batch ...int) (sql.Result, error) + Insert(table string, data interface{}, batch ...int) (sql.Result, error) // See Core.Insert. + InsertIgnore(table string, data interface{}, batch ...int) (sql.Result, error) // See Core.InsertIgnore. + Replace(table string, data interface{}, batch ...int) (sql.Result, error) // See Core.Replace. + Save(table string, data interface{}, batch ...int) (sql.Result, error) // See Core.Save. - BatchInsert(table string, list interface{}, batch ...int) (sql.Result, error) - BatchReplace(table string, list interface{}, batch ...int) (sql.Result, error) - BatchSave(table string, list interface{}, batch ...int) (sql.Result, error) + BatchInsert(table string, list interface{}, batch ...int) (sql.Result, error) // See Core.BatchInsert. + BatchReplace(table string, list interface{}, batch ...int) (sql.Result, error) // See Core.BatchReplace. + BatchSave(table string, list interface{}, batch ...int) (sql.Result, error) // See Core.BatchSave. - Update(table string, data interface{}, condition interface{}, args ...interface{}) (sql.Result, error) - Delete(table string, condition interface{}, args ...interface{}) (sql.Result, error) + Update(table string, data interface{}, condition interface{}, args ...interface{}) (sql.Result, error) // See Core.Update. + Delete(table string, condition interface{}, args ...interface{}) (sql.Result, error) // See Core.Delete. // =========================================================================== // Internal APIs for CURD, which can be overwrote for custom CURD implements. // =========================================================================== - DoQuery(link Link, sql string, args ...interface{}) (rows *sql.Rows, err error) - DoGetAll(link Link, sql string, args ...interface{}) (result Result, err error) - DoExec(link Link, sql string, args ...interface{}) (result sql.Result, err error) - DoPrepare(link Link, sql string) (*Stmt, error) - DoInsert(link Link, table string, data interface{}, option int, batch ...int) (result sql.Result, err error) - DoBatchInsert(link Link, table string, list interface{}, option int, batch ...int) (result sql.Result, err error) - DoUpdate(link Link, table string, data interface{}, condition string, args ...interface{}) (result sql.Result, err error) - DoDelete(link Link, table string, condition string, args ...interface{}) (result sql.Result, err error) + DoQuery(link Link, sql string, args ...interface{}) (rows *sql.Rows, err error) // See Core.DoQuery. + DoGetAll(link Link, sql string, args ...interface{}) (result Result, err error) // See Core.DoGetAll. + DoExec(link Link, sql string, args ...interface{}) (result sql.Result, err error) // See Core.DoExec. + DoPrepare(link Link, sql string) (*Stmt, error) // See Core.DoPrepare. + DoInsert(link Link, table string, data interface{}, option int, batch ...int) (result sql.Result, err error) // See Core.DoInsert. + DoBatchInsert(link Link, table string, list interface{}, option int, batch ...int) (result sql.Result, err error) // See Core.DoBatchInsert. + DoUpdate(link Link, table string, data interface{}, condition string, args ...interface{}) (result sql.Result, err error) // See Core.DoUpdate. + DoDelete(link Link, table string, condition string, args ...interface{}) (result sql.Result, err error) // See Core.DoDelete. // =========================================================================== // Query APIs for convenience purpose. // =========================================================================== - GetAll(sql string, args ...interface{}) (Result, error) - GetOne(sql string, args ...interface{}) (Record, error) - GetValue(sql string, args ...interface{}) (Value, error) - GetArray(sql string, args ...interface{}) ([]Value, error) - GetCount(sql string, args ...interface{}) (int, error) - GetStruct(objPointer interface{}, sql string, args ...interface{}) error - GetStructs(objPointerSlice interface{}, sql string, args ...interface{}) error - GetScan(objPointer interface{}, sql string, args ...interface{}) error + GetAll(sql string, args ...interface{}) (Result, error) // See Core.GetAll. + GetOne(sql string, args ...interface{}) (Record, error) // See Core.GetOne. + GetValue(sql string, args ...interface{}) (Value, error) // See Core.GetValue. + GetArray(sql string, args ...interface{}) ([]Value, error) // See Core.GetArray. + GetCount(sql string, args ...interface{}) (int, error) // See Core.GetCount. + GetStruct(objPointer interface{}, sql string, args ...interface{}) error // See Core.GetStruct. + GetStructs(objPointerSlice interface{}, sql string, args ...interface{}) error // See Core.GetStructs. + GetScan(objPointer interface{}, sql string, args ...interface{}) error // See Core.GetScan. // =========================================================================== // Master/Slave specification support. // =========================================================================== - Master() (*sql.DB, error) - Slave() (*sql.DB, error) + Master() (*sql.DB, error) // See Core.Master. + Slave() (*sql.DB, error) // See Core.Slave. // =========================================================================== // Ping-Pong. // =========================================================================== - PingMaster() error - PingSlave() error + PingMaster() error // See Core.PingMaster. + PingSlave() error // See Core.PingSlave. // =========================================================================== // Transaction. // =========================================================================== - Begin() (*TX, error) - Transaction(f func(tx *TX) error) (err error) + Begin() (*TX, error) // See Core.Begin. + Transaction(f func(tx *TX) error) (err error) // See Core.Transaction. // =========================================================================== // Configuration methods. // =========================================================================== - GetCache() *gcache.Cache - SetDebug(debug bool) - GetDebug() bool - SetSchema(schema string) - GetSchema() string - GetPrefix() string - GetGroup() string - SetDryRun(dryrun bool) - GetDryRun() bool - SetLogger(logger *glog.Logger) - GetLogger() *glog.Logger - GetConfig() *ConfigNode - SetMaxIdleConnCount(n int) - SetMaxOpenConnCount(n int) - SetMaxConnLifetime(d time.Duration) + GetCache() *gcache.Cache // See Core.GetCache. + SetDebug(debug bool) // See Core.SetDebug. + GetDebug() bool // See Core.GetDebug. + SetSchema(schema string) // See Core.SetSchema. + GetSchema() string // See Core.GetSchema. + GetPrefix() string // See Core.GetPrefix. + GetGroup() string // See Core.GetGroup. + SetDryRun(enabled bool) // See Core.SetDryRun. + GetDryRun() bool // See Core.GetDryRun. + SetLogger(logger *glog.Logger) // See Core.SetLogger. + GetLogger() *glog.Logger // See Core.GetLogger. + GetConfig() *ConfigNode // See Core.GetConfig. + SetMaxIdleConnCount(n int) // See Core.SetMaxIdleConnCount. + SetMaxOpenConnCount(n int) // See Core.SetMaxOpenConnCount. + SetMaxConnLifeTime(d time.Duration) // See Core.SetMaxConnLifeTime. + SetMaxConnIdleTime(d time.Duration) // See Core.SetMaxConnIdleTime // =========================================================================== // Utility methods. // =========================================================================== - GetCtx() context.Context - GetChars() (charLeft string, charRight string) - GetMaster(schema ...string) (*sql.DB, error) - GetSlave(schema ...string) (*sql.DB, error) - QuoteWord(s string) string - QuoteString(s string) string - QuotePrefixTableName(table string) string - Tables(schema ...string) (tables []string, err error) - TableFields(table string, schema ...string) (map[string]*TableField, error) - HasTable(name string) (bool, error) - FilteredLinkInfo() string + GetCtx() context.Context // See Core.GetCtx. + GetChars() (charLeft string, charRight string) // See Core.GetChars. + GetMaster(schema ...string) (*sql.DB, error) // See Core.GetMaster. + GetSlave(schema ...string) (*sql.DB, error) // See Core.GetSlave. + QuoteWord(s string) string // See Core.QuoteWord. + QuoteString(s string) string // See Core.QuoteString. + QuotePrefixTableName(table string) string // See Core.QuotePrefixTableName. + Tables(schema ...string) (tables []string, err error) // See Core.Tables. + TableFields(link Link, table string, schema ...string) (map[string]*TableField, error) // See Core.TableFields. + HasTable(name string) (bool, error) // See Core.HasTable. + FilteredLinkInfo() string // See Core.FilteredLinkInfo. // HandleSqlBeforeCommit is a hook function, which deals with the sql string before // it's committed to underlying driver. The parameter `link` specifies the current // database connection operation object. You can modify the sql string `sql` and its // arguments `args` as you wish before they're committed to driver. + // Also see Core.HandleSqlBeforeCommit. HandleSqlBeforeCommit(link Link, sql string, args []interface{}) (string, []interface{}) // =========================================================================== // Internal methods, for internal usage purpose, you do not need consider it. // =========================================================================== - mappingAndFilterData(schema, table string, data map[string]interface{}, filter bool) (map[string]interface{}, error) - convertFieldValueToLocalValue(fieldValue interface{}, fieldType string) interface{} - convertRowsToResult(rows *sql.Rows) (Result, error) + mappingAndFilterData(schema, table string, data map[string]interface{}, filter bool) (map[string]interface{}, error) // See Core.mappingAndFilterData. + convertFieldValueToLocalValue(fieldValue interface{}, fieldType string) interface{} // See Core.convertFieldValueToLocalValue. + convertRowsToResult(rows *sql.Rows) (Result, error) // See Core.convertRowsToResult. } // Core is the base struct for database management. @@ -299,6 +307,9 @@ var ( // internalCache is the memory cache for internal usage. internalCache = gcache.New() + // tableFieldsMap caches the table information retrived from database. + tableFieldsMap = gmap.New(true) + // allDryRun sets dry-run feature for all database connections. // It is commonly used for command options for convenience. allDryRun = false @@ -471,15 +482,26 @@ func (c *Core) getSqlDb(master bool, schema ...string) (sqlDb *sql.DB, err error } // Cache the underlying connection pool object by node. v, _ := internalCache.GetOrSetFuncLock(node.String(), func() (interface{}, error) { - sqlDb, err = c.db.Open(node) - if err != nil { - intlog.Printf(`db open failed: %v, %+v`, err, node) - return nil, err - } intlog.Printf( - `open new connection, master:%v, config:%+v, node:%+v`, + `open new connection, master:%#v, config:%#v, node:%#v`, master, c.config, node, ) + defer func() { + if err != nil { + intlog.Printf(`open new connection failed: %v, %#v`, err, node) + } else { + intlog.Printf( + `open new connection success, master:%#v, config:%#v, node:%#v`, + master, c.config, node, + ) + } + }() + + sqlDb, err = c.db.Open(node) + if err != nil { + return nil, err + } + if c.config.MaxIdleConnCount > 0 { sqlDb.SetMaxIdleConns(c.config.MaxIdleConnCount) } else { @@ -490,17 +512,24 @@ func (c *Core) getSqlDb(master bool, schema ...string) (sqlDb *sql.DB, err error } else { sqlDb.SetMaxOpenConns(defaultMaxOpenConnCount) } - if c.config.MaxConnLifetime > 0 { + if c.config.MaxConnLifeTime > 0 { // Automatically checks whether MaxConnLifetime is configured using string like: "30s", "60s", etc. // Or else it is configured just using number, which means value in seconds. - if c.config.MaxConnLifetime > time.Second { - sqlDb.SetConnMaxLifetime(c.config.MaxConnLifetime) + if c.config.MaxConnLifeTime > time.Second { + sqlDb.SetConnMaxLifetime(c.config.MaxConnLifeTime) } else { - sqlDb.SetConnMaxLifetime(c.config.MaxConnLifetime * time.Second) + sqlDb.SetConnMaxLifetime(c.config.MaxConnLifeTime * time.Second) } } else { sqlDb.SetConnMaxLifetime(defaultMaxConnLifeTime) } + if c.config.MaxConnIdleTime > 0 { + if c.config.MaxConnIdleTime > time.Second { + sqlDb.SetConnMaxIdleTime(c.config.MaxConnIdleTime) + } else { + sqlDb.SetConnMaxIdleTime(c.config.MaxConnIdleTime * time.Second) + } + } return sqlDb, nil }, 0) if v != nil && sqlDb == nil { diff --git a/database/gdb/gdb_core.go b/database/gdb/gdb_core.go index 7f2a6a1b5..8265c540a 100644 --- a/database/gdb/gdb_core.go +++ b/database/gdb/gdb_core.go @@ -113,7 +113,6 @@ func (c *Core) DoQuery(link Link, sql string, args ...interface{}) (rows *sql.Ro if c.GetConfig().QueryTimeout > 0 { ctx, _ = context.WithTimeout(ctx, c.GetConfig().QueryTimeout) } - mTime1 := gtime.TimestampMilli() rows, err = link.QueryContext(ctx, sql, args...) mTime2 := gtime.TimestampMilli() @@ -277,7 +276,7 @@ func (c *Core) GetOne(sql string, args ...interface{}) (Record, error) { } // GetArray queries and returns data values as slice from database. -// Note that if there're multiple columns in the result, it returns just one column values randomly. +// Note that if there are multiple columns in the result, it returns just one column values randomly. func (c *Core) GetArray(sql string, args ...interface{}) ([]Value, error) { all, err := c.db.DoGetAll(nil, sql, args...) if err != nil { @@ -382,13 +381,13 @@ func (c *Core) Begin() (*TX, error) { if master, err := c.db.Master(); err != nil { return nil, err } else { - ctx := c.db.GetCtx() - if c.GetConfig().TranTimeout > 0 { - var cancelFunc context.CancelFunc - ctx, cancelFunc = context.WithTimeout(ctx, c.GetConfig().TranTimeout) - defer cancelFunc() - } - if tx, err := master.BeginTx(ctx, nil); err == nil { + //ctx := c.db.GetCtx() + //if c.GetConfig().TranTimeout > 0 { + // var cancelFunc context.CancelFunc + // ctx, cancelFunc = context.WithTimeout(ctx, c.GetConfig().TranTimeout) + // defer cancelFunc() + //} + if tx, err := master.Begin(); err == nil { return &TX{ db: c.db, tx: tx, diff --git a/database/gdb/gdb_core_config.go b/database/gdb/gdb_core_config.go index ab222cf44..a5df3c52a 100644 --- a/database/gdb/gdb_core_config.go +++ b/database/gdb/gdb_core_config.go @@ -42,7 +42,8 @@ type ConfigNode struct { LinkInfo string `json:"link"` // (Optional) Custom link information, when it is used, configuration Host/Port/User/Pass/Name are ignored. MaxIdleConnCount int `json:"maxIdle"` // (Optional) Max idle connection configuration for underlying connection pool. MaxOpenConnCount int `json:"maxOpen"` // (Optional) Max open connection configuration for underlying connection pool. - MaxConnLifetime time.Duration `json:"maxLifetime"` // (Optional) Max connection TTL configuration for underlying connection pool. + MaxConnIdleTime time.Duration `json:"maxIdleTime"` // (Optional) Max connection TTL configuration for underlying connection pool. + MaxConnLifeTime time.Duration `json:"maxLifeTime"` // (Optional) Max amount of time a connection may be idle before being closed. QueryTimeout time.Duration `json:"queryTimeout"` // (Optional) Max query time for per dql. ExecTimeout time.Duration `json:"execTimeout"` // (Optional) Max exec time for dml. TranTimeout time.Duration `json:"tranTimeout"` // (Optional) Max exec time time for a transaction. @@ -141,31 +142,60 @@ func (c *Core) GetLogger() *glog.Logger { return c.logger } -// SetMaxIdleConnCount sets the max idle connection count for underlying connection pool. +// SetMaxIdleConnCount sets the maximum number of connections in the idle +// connection pool. +// +// If MaxOpenConns is greater than 0 but less than the new MaxIdleConns, +// then the new MaxIdleConns will be reduced to match the MaxOpenConns limit. +// +// If n <= 0, no idle connections are retained. +// +// The default max idle connections is currently 2. This may change in +// a future release. func (c *Core) SetMaxIdleConnCount(n int) { c.config.MaxIdleConnCount = n } -// SetMaxOpenConnCount sets the max open connection count for underlying connection pool. +// SetMaxOpenConnCount sets the maximum number of open connections to the database. +// +// If MaxIdleConns is greater than 0 and the new MaxOpenConns is less than +// MaxIdleConns, then MaxIdleConns will be reduced to match the new +// MaxOpenConns limit. +// +// If n <= 0, then there is no limit on the number of open connections. +// The default is 0 (unlimited). func (c *Core) SetMaxOpenConnCount(n int) { c.config.MaxOpenConnCount = n } -// SetMaxConnLifetime sets the connection TTL for underlying connection pool. -// If parameter `d` <= 0, it means the connection never expires. -func (c *Core) SetMaxConnLifetime(d time.Duration) { - c.config.MaxConnLifetime = d +// SetMaxConnIdleTime sets the maximum amount of time a connection may be idle. +// +// Expired connections may be closed lazily before reuse. +// +// If d <= 0, connections are not closed due to a connection's idle time. +func (c *Core) SetMaxConnIdleTime(d time.Duration) { + c.config.MaxConnIdleTime = d +} + +// SetMaxConnLifeTime sets the maximum amount of time a connection may be reused. +// +// Expired connections may be closed lazily before reuse. +// +// If d <= 0, connections are not closed due to a connection's age. +func (c *Core) SetMaxConnLifeTime(d time.Duration) { + c.config.MaxConnLifeTime = d } // String returns the node as string. func (node *ConfigNode) String() string { return fmt.Sprintf( - `%s@%s:%s,%s,%s,%s,%s,%v,%d-%d-%d#%s`, + `%s@%s:%s,%s,%s,%s,%s,%v,%d-%d-%d-%d#%s`, node.User, node.Host, node.Port, node.Name, node.Type, node.Role, node.Charset, node.Debug, node.MaxIdleConnCount, node.MaxOpenConnCount, - node.MaxConnLifetime, + node.MaxConnIdleTime, + node.MaxConnLifeTime, node.LinkInfo, ) } diff --git a/database/gdb/gdb_core_structure.go b/database/gdb/gdb_core_structure.go index de5b9e770..8e28f239b 100644 --- a/database/gdb/gdb_core_structure.go +++ b/database/gdb/gdb_core_structure.go @@ -149,7 +149,7 @@ func (c *Core) convertFieldValueToLocalValue(fieldValue interface{}, fieldType s // mappingAndFilterData automatically mappings the map key to table field and removes // all key-value pairs that are not the field of given table. func (c *Core) mappingAndFilterData(schema, table string, data map[string]interface{}, filter bool) (map[string]interface{}, error) { - if fieldsMap, err := c.db.TableFields(table, schema); err == nil { + if fieldsMap, err := c.db.TableFields(nil, table, schema); err == nil { fieldsKeyMap := make(map[string]interface{}, len(fieldsMap)) for k, _ := range fieldsMap { fieldsKeyMap[k] = nil diff --git a/database/gdb/gdb_driver_mssql.go b/database/gdb/gdb_driver_mssql.go index 6ba855489..f288187c9 100644 --- a/database/gdb/gdb_driver_mssql.go +++ b/database/gdb/gdb_driver_mssql.go @@ -203,7 +203,9 @@ func (d *DriverMssql) Tables(schema ...string) (tables []string, err error) { } // TableFields retrieves and returns the fields information of specified table of current schema. -func (d *DriverMssql) TableFields(table string, schema ...string) (fields map[string]*TableField, err error) { +// +// Also see DriverMysql.TableFields. +func (d *DriverMssql) TableFields(link Link, table string, schema ...string) (fields map[string]*TableField, err error) { charL, charR := d.GetChars() table = gstr.Trim(table, charL+charR) if gstr.Contains(table, " ") { @@ -213,18 +215,21 @@ func (d *DriverMssql) TableFields(table string, schema ...string) (fields map[st if len(schema) > 0 && schema[0] != "" { checkSchema = schema[0] } - v, _ := internalCache.GetOrSetFunc( - fmt.Sprintf(`mssql_table_fields_%s_%s@group:%s`, table, checkSchema, d.GetGroup()), - func() (interface{}, error) { - var ( - result Result - link *sql.DB - ) + tableFieldsCacheKey := fmt.Sprintf( + `mssql_table_fields_%s_%s@group:%s`, + table, checkSchema, d.GetGroup(), + ) + v := tableFieldsMap.GetOrSetFuncLock(tableFieldsCacheKey, func() interface{} { + var ( + result Result + ) + if link == nil { link, err = d.db.GetSlave(checkSchema) if err != nil { - return nil, err + return nil } - structureSql := fmt.Sprintf(` + } + structureSql := fmt.Sprintf(` SELECT a.name Field, CASE b.name @@ -252,29 +257,29 @@ LEFT JOIN sys.extended_properties g ON a.id=g.major_id AND a.colid=g.minor_id LEFT JOIN sys.extended_properties f ON d.id=f.major_id AND f.minor_id =0 WHERE d.name='%s' ORDER BY a.id,a.colorder`, - strings.ToUpper(table), - ) - structureSql, _ = gregex.ReplaceString(`[\n\r\s]+`, " ", gstr.Trim(structureSql)) - result, err = d.db.DoGetAll(link, structureSql) - if err != nil { - return nil, err + strings.ToUpper(table), + ) + structureSql, _ = gregex.ReplaceString(`[\n\r\s]+`, " ", gstr.Trim(structureSql)) + result, err = d.db.DoGetAll(link, structureSql) + if err != nil { + return nil + } + fields = make(map[string]*TableField) + for i, m := range result { + fields[strings.ToLower(m["Field"].String())] = &TableField{ + Index: i, + Name: strings.ToLower(m["Field"].String()), + Type: strings.ToLower(m["Type"].String()), + Null: m["Null"].Bool(), + Key: m["Key"].String(), + Default: m["Default"].Val(), + Extra: m["Extra"].String(), + Comment: m["Comment"].String(), } - fields = make(map[string]*TableField) - for i, m := range result { - fields[strings.ToLower(m["Field"].String())] = &TableField{ - Index: i, - Name: strings.ToLower(m["Field"].String()), - Type: strings.ToLower(m["Type"].String()), - Null: m["Null"].Bool(), - Key: m["Key"].String(), - Default: m["Default"].Val(), - Extra: m["Extra"].String(), - Comment: m["Comment"].String(), - } - } - return fields, nil - }, 0) - if err == nil { + } + return fields + }) + if v != nil { fields = v.(map[string]*TableField) } return diff --git a/database/gdb/gdb_driver_mysql.go b/database/gdb/gdb_driver_mysql.go index 5ac100350..9f6f6bf45 100644 --- a/database/gdb/gdb_driver_mysql.go +++ b/database/gdb/gdb_driver_mysql.go @@ -102,13 +102,16 @@ func (d *DriverMysql) Tables(schema ...string) (tables []string, err error) { // TableFields retrieves and returns the fields information of specified table of current // schema. // +// The parameter `link` is optional, if given nil it automatically retrieves a raw sql connection +// as its link to proceed necessary sql query. +// // Note that it returns a map containing the field name and its corresponding fields. // As a map is unsorted, the TableField struct has a "Index" field marks its sequence in // the fields. // // It's using cache feature to enhance the performance, which is never expired util the // process restarts. -func (d *DriverMysql) TableFields(table string, schema ...string) (fields map[string]*TableField, err error) { +func (d *DriverMysql) TableFields(link Link, table string, schema ...string) (fields map[string]*TableField, err error) { charL, charR := d.GetChars() table = gstr.Trim(table, charL+charR) if gstr.Contains(table, " ") { @@ -118,40 +121,43 @@ func (d *DriverMysql) TableFields(table string, schema ...string) (fields map[st if len(schema) > 0 && schema[0] != "" { checkSchema = schema[0] } - v, _ := internalCache.GetOrSetFunc( - fmt.Sprintf(`mysql_table_fields_%s_%s@group:%s`, table, checkSchema, d.GetGroup()), - func() (interface{}, error) { - var ( - result Result - link *sql.DB - ) + tableFieldsCacheKey := fmt.Sprintf( + `mysql_table_fields_%s_%s@group:%s`, + table, checkSchema, d.GetGroup(), + ) + v := tableFieldsMap.GetOrSetFuncLock(tableFieldsCacheKey, func() interface{} { + var ( + result Result + ) + if link == nil { link, err = d.db.GetSlave(checkSchema) if err != nil { - return nil, err + return nil } - result, err = d.db.DoGetAll( - link, - fmt.Sprintf(`SHOW FULL COLUMNS FROM %s`, d.db.QuoteWord(table)), - ) - if err != nil { - return nil, err + } + result, err = d.db.DoGetAll( + link, + fmt.Sprintf(`SHOW FULL COLUMNS FROM %s`, d.db.QuoteWord(table)), + ) + if err != nil { + return nil + } + fields = make(map[string]*TableField) + for i, m := range result { + fields[m["Field"].String()] = &TableField{ + Index: i, + Name: m["Field"].String(), + Type: m["Type"].String(), + Null: m["Null"].Bool(), + Key: m["Key"].String(), + Default: m["Default"].Val(), + Extra: m["Extra"].String(), + Comment: m["Comment"].String(), } - fields = make(map[string]*TableField) - for i, m := range result { - fields[m["Field"].String()] = &TableField{ - Index: i, - Name: m["Field"].String(), - Type: m["Type"].String(), - Null: m["Null"].Bool(), - Key: m["Key"].String(), - Default: m["Default"].Val(), - Extra: m["Extra"].String(), - Comment: m["Comment"].String(), - } - } - return fields, nil - }, 0) - if err == nil { + } + return fields + }) + if v != nil { fields = v.(map[string]*TableField) } return diff --git a/database/gdb/gdb_driver_oracle.go b/database/gdb/gdb_driver_oracle.go index e0bac602f..f8011b91a 100644 --- a/database/gdb/gdb_driver_oracle.go +++ b/database/gdb/gdb_driver_oracle.go @@ -179,7 +179,9 @@ func (d *DriverOracle) Tables(schema ...string) (tables []string, err error) { } // TableFields retrieves and returns the fields information of specified table of current schema. -func (d *DriverOracle) TableFields(table string, schema ...string) (fields map[string]*TableField, err error) { +// +// Also see DriverMysql.TableFields. +func (d *DriverOracle) TableFields(link Link, table string, schema ...string) (fields map[string]*TableField, err error) { charL, charR := d.GetChars() table = gstr.Trim(table, charL+charR) if gstr.Contains(table, " ") { @@ -189,11 +191,14 @@ func (d *DriverOracle) TableFields(table string, schema ...string) (fields map[s if len(schema) > 0 && schema[0] != "" { checkSchema = schema[0] } - v, _ := internalCache.GetOrSetFunc( - fmt.Sprintf(`oracle_table_fields_%s_%s@group:%s`, table, checkSchema, d.GetGroup()), - func() (interface{}, error) { - result := (Result)(nil) - structureSql := fmt.Sprintf(` + tableFieldsCacheKey := fmt.Sprintf( + `oracle_table_fields_%s_%s@group:%s`, + table, checkSchema, d.GetGroup(), + ) + v := tableFieldsMap.GetOrSetFuncLock(tableFieldsCacheKey, func() interface{} { + var ( + result Result + structureSql = fmt.Sprintf(` SELECT COLUMN_NAME AS FIELD, CASE DATA_TYPE @@ -203,22 +208,29 @@ SELECT FROM USER_TAB_COLUMNS WHERE TABLE_NAME = '%s' ORDER BY COLUMN_ID`, strings.ToUpper(table), ) - structureSql, _ = gregex.ReplaceString(`[\n\r\s]+`, " ", gstr.Trim(structureSql)) - result, err = d.db.GetAll(structureSql) + ) + structureSql, _ = gregex.ReplaceString(`[\n\r\s]+`, " ", gstr.Trim(structureSql)) + if link == nil { + link, err = d.db.GetSlave(checkSchema) if err != nil { - return nil, err + return nil } - fields = make(map[string]*TableField) - for i, m := range result { - fields[strings.ToLower(m["FIELD"].String())] = &TableField{ - Index: i, - Name: strings.ToLower(m["FIELD"].String()), - Type: strings.ToLower(m["TYPE"].String()), - } + } + result, err = d.db.DoGetAll(link, structureSql) + if err != nil { + return nil + } + fields = make(map[string]*TableField) + for i, m := range result { + fields[strings.ToLower(m["FIELD"].String())] = &TableField{ + Index: i, + Name: strings.ToLower(m["FIELD"].String()), + Type: strings.ToLower(m["TYPE"].String()), } - return fields, nil - }, 0) - if err == nil { + } + return fields + }) + if v != nil { fields = v.(map[string]*TableField) } return diff --git a/database/gdb/gdb_driver_pgsql.go b/database/gdb/gdb_driver_pgsql.go index a047733a2..95cf65087 100644 --- a/database/gdb/gdb_driver_pgsql.go +++ b/database/gdb/gdb_driver_pgsql.go @@ -111,7 +111,9 @@ func (d *DriverPgsql) Tables(schema ...string) (tables []string, err error) { } // TableFields retrieves and returns the fields information of specified table of current schema. -func (d *DriverPgsql) TableFields(table string, schema ...string) (fields map[string]*TableField, err error) { +// +// Also see DriverMysql.TableFields. +func (d *DriverPgsql) TableFields(link Link, table string, schema ...string) (fields map[string]*TableField, err error) { charL, charR := d.GetChars() table = gstr.Trim(table, charL+charR) if gstr.Contains(table, " ") { @@ -122,41 +124,43 @@ func (d *DriverPgsql) TableFields(table string, schema ...string) (fields map[st if len(schema) > 0 && schema[0] != "" { checkSchema = schema[0] } - v, _ := internalCache.GetOrSetFunc( - fmt.Sprintf(`pgsql_table_fields_%s_%s@group:%s`, table, checkSchema, d.GetGroup()), - func() (interface{}, error) { - var ( - result Result - link *sql.DB - ) - link, err = d.db.GetSlave(checkSchema) - if err != nil { - return nil, err - } - structureSql := fmt.Sprintf(` + tableFieldsCacheKey := fmt.Sprintf( + `pgsql_table_fields_%s_%s@group:%s`, + table, checkSchema, d.GetGroup(), + ) + v := tableFieldsMap.GetOrSetFuncLock(tableFieldsCacheKey, func() interface{} { + var ( + result Result + structureSql = fmt.Sprintf(` SELECT a.attname AS field, t.typname AS type FROM pg_class c, pg_attribute a LEFT OUTER JOIN pg_description b ON a.attrelid=b.objoid AND a.attnum = b.objsubid,pg_type t WHERE c.relname = '%s' and a.attnum > 0 and a.attrelid = c.oid and a.atttypid = t.oid ORDER BY a.attnum`, strings.ToLower(table), ) - structureSql, _ = gregex.ReplaceString(`[\n\r\s]+`, " ", gstr.Trim(structureSql)) - result, err = d.db.DoGetAll(link, structureSql) + ) + structureSql, _ = gregex.ReplaceString(`[\n\r\s]+`, " ", gstr.Trim(structureSql)) + if link == nil { + link, err = d.db.GetSlave(checkSchema) if err != nil { - return nil, err + return nil } - - fields = make(map[string]*TableField) - for i, m := range result { - fields[m["field"].String()] = &TableField{ - Index: i, - Name: m["field"].String(), - Type: m["type"].String(), - } + } + result, err = d.db.DoGetAll(link, structureSql) + if err != nil { + return nil + } + fields = make(map[string]*TableField) + for i, m := range result { + fields[m["field"].String()] = &TableField{ + Index: i, + Name: m["field"].String(), + Type: m["type"].String(), } - return fields, nil - }, 0) - if err == nil { + } + return fields + }) + if v != nil { fields = v.(map[string]*TableField) } return diff --git a/database/gdb/gdb_driver_sqlite.go b/database/gdb/gdb_driver_sqlite.go index ed577217a..ba000fbbb 100644 --- a/database/gdb/gdb_driver_sqlite.go +++ b/database/gdb/gdb_driver_sqlite.go @@ -93,7 +93,9 @@ func (d *DriverSqlite) Tables(schema ...string) (tables []string, err error) { } // TableFields retrieves and returns the fields information of specified table of current schema. -func (d *DriverSqlite) TableFields(table string, schema ...string) (fields map[string]*TableField, err error) { +// +// Also see DriverMysql.TableFields. +func (d *DriverSqlite) TableFields(link Link, table string, schema ...string) (fields map[string]*TableField, err error) { charL, charR := d.GetChars() table = gstr.Trim(table, charL+charR) if gstr.Contains(table, " ") { @@ -103,32 +105,35 @@ func (d *DriverSqlite) TableFields(table string, schema ...string) (fields map[s if len(schema) > 0 && schema[0] != "" { checkSchema = schema[0] } - v, _ := internalCache.GetOrSetFunc( - fmt.Sprintf(`sqlite_table_fields_%s_%s@group:%s`, table, checkSchema, d.GetGroup()), - func() (interface{}, error) { - var ( - result Result - link *sql.DB - ) + tableFieldsCacheKey := fmt.Sprintf( + `sqlite_table_fields_%s_%s@group:%s`, + table, checkSchema, d.GetGroup(), + ) + v := tableFieldsMap.GetOrSetFuncLock(tableFieldsCacheKey, func() interface{} { + var ( + result Result + ) + if link == nil { link, err = d.db.GetSlave(checkSchema) if err != nil { - return nil, err + return nil } - result, err = d.db.DoGetAll(link, fmt.Sprintf(`PRAGMA TABLE_INFO(%s)`, table)) - if err != nil { - return nil, err + } + result, err = d.db.DoGetAll(link, fmt.Sprintf(`PRAGMA TABLE_INFO(%s)`, table)) + if err != nil { + return nil + } + fields = make(map[string]*TableField) + for i, m := range result { + fields[strings.ToLower(m["name"].String())] = &TableField{ + Index: i, + Name: strings.ToLower(m["name"].String()), + Type: strings.ToLower(m["type"].String()), } - fields = make(map[string]*TableField) - for i, m := range result { - fields[strings.ToLower(m["name"].String())] = &TableField{ - Index: i, - Name: strings.ToLower(m["name"].String()), - Type: strings.ToLower(m["type"].String()), - } - } - return fields, nil - }, 0) - if err == nil { + } + return fields + }) + if v != nil { fields = v.(map[string]*TableField) } return diff --git a/database/gdb/gdb_model_fields.go b/database/gdb/gdb_model_fields.go index bbcb5978f..c10e00461 100644 --- a/database/gdb/gdb_model_fields.go +++ b/database/gdb/gdb_model_fields.go @@ -94,7 +94,7 @@ func (m *Model) GetFieldsStr(prefix ...string) string { if len(prefix) > 0 { prefixStr = prefix[0] } - tableFields, err := m.db.TableFields(m.tables) + tableFields, err := m.TableFields(m.tables) if err != nil { panic(err) } @@ -131,7 +131,7 @@ func (m *Model) GetFieldsExStr(fields string, prefix ...string) string { if len(prefix) > 0 { prefixStr = prefix[0] } - tableFields, err := m.db.TableFields(m.tables) + tableFields, err := m.TableFields(m.tables) if err != nil { panic(err) } @@ -159,7 +159,7 @@ func (m *Model) GetFieldsExStr(fields string, prefix ...string) string { // HasField determine whether the field exists in the table. func (m *Model) HasField(field string) (bool, error) { - tableFields, err := m.db.TableFields(m.tables) + tableFields, err := m.TableFields(m.tables) if err != nil { return false, err } diff --git a/database/gdb/gdb_model_select.go b/database/gdb/gdb_model_select.go index 1cd5139fa..962592b33 100644 --- a/database/gdb/gdb_model_select.go +++ b/database/gdb/gdb_model_select.go @@ -84,7 +84,7 @@ func (m *Model) getFieldsFiltered() string { panic("function FieldsEx supports only single table operations") } // Filter table fields with fieldEx. - tableFields, err := m.db.TableFields(m.tables) + tableFields, err := m.TableFields(m.tables) if err != nil { panic(err) } diff --git a/database/gdb/gdb_model_time.go b/database/gdb/gdb_model_time.go index ce14d831e..e167fbb35 100644 --- a/database/gdb/gdb_model_time.go +++ b/database/gdb/gdb_model_time.go @@ -93,7 +93,7 @@ func (m *Model) getSoftFieldNameDeleted(table ...string) (field string) { // getSoftFieldName retrieves and returns the field name of the table for possible key. func (m *Model) getSoftFieldName(table string, keys []string) (field string) { - fieldsMap, _ := m.db.TableFields(table) + fieldsMap, _ := m.TableFields(table) if len(fieldsMap) > 0 { for _, key := range keys { field, _ = gutil.MapPossibleItemByKey( diff --git a/database/gdb/gdb_model_utility.go b/database/gdb/gdb_model_utility.go index 93ede1a04..93890326b 100644 --- a/database/gdb/gdb_model_utility.go +++ b/database/gdb/gdb_model_utility.go @@ -28,12 +28,31 @@ func (m *Model) getModel() *Model { } } +// TableFields retrieves and returns the fields information of specified table of current +// schema. +// +// Also see DriverMysql.TableFields. +func (m *Model) TableFields(table string, schema ...string) (fields map[string]*TableField, err error) { + var ( + link Link + ) + if m.tx != nil { + link = m.tx.tx + } else { + link, err = m.db.GetSlave(schema...) + if err != nil { + return + } + } + return m.db.TableFields(link, table, schema...) +} + // mappingAndFilterToTableFields mappings and changes given field name to really table field name. // Eg: // ID -> id // NICK_Name -> nickname func (m *Model) mappingAndFilterToTableFields(fields []string, filter bool) []string { - fieldsMap, err := m.db.TableFields(m.tables) + fieldsMap, err := m.TableFields(m.tables) if err != nil || len(fieldsMap) == 0 { return fields } @@ -188,7 +207,7 @@ func (m *Model) getLink(master bool) Link { // "user", "user u", "user as u, user_detail as ud". func (m *Model) getPrimaryKey() string { table := gstr.SplitAndTrim(m.tables, " ")[0] - tableFields, err := m.db.TableFields(table) + tableFields, err := m.TableFields(table) if err != nil { return "" } diff --git a/database/gdb/gdb_z_init_test.go b/database/gdb/gdb_z_init_test.go index fe9fcc47a..abe205257 100644 --- a/database/gdb/gdb_z_init_test.go +++ b/database/gdb/gdb_z_init_test.go @@ -52,7 +52,7 @@ func init() { Weight: 1, MaxIdleConnCount: 10, MaxOpenConnCount: 10, - MaxConnLifetime: 600, + MaxConnLifeTime: 600, } nodePrefix := configNode nodePrefix.Prefix = TableNamePrefix1 diff --git a/database/gdb/gdb_z_mysql_internal_test.go b/database/gdb/gdb_z_mysql_internal_test.go index 736a1c1f0..ed80711a6 100644 --- a/database/gdb/gdb_z_mysql_internal_test.go +++ b/database/gdb/gdb_z_mysql_internal_test.go @@ -45,7 +45,7 @@ func init() { Weight: 1, MaxIdleConnCount: 10, MaxOpenConnCount: 10, - MaxConnLifetime: 600, + MaxConnLifeTime: 600, } AddConfigNode(DefaultGroupName, configNode) // Default db. diff --git a/version.go b/version.go index 04fce82b3..189fb2e19 100644 --- a/version.go +++ b/version.go @@ -1,4 +1,4 @@ package gf -const VERSION = "v1.15.5" +const VERSION = "v1.15.6" const AUTHORS = "john"