bingyi/gf
1
0
Fork 0
mirror of https://gitee.com/johng/gf synced 2026-06-06 16:21:40 +08:00
Code Issues Packages Projects Releases Wiki Activity

improve data converting for DB.DoInsert/DoUpdate (#2830)

Browse Source
This commit is contained in:
John Guo
2023-08-07 21:03:56 +08:00
committed by GitHub
parent 5de2cfbfa1
commit 7c1be3eb63
8 changed files with 127 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
contrib/drivers/clickhouse/clickhouse.go
View File

@ -18,6 +18,9 @@ import (
"time"
"github.com/ClickHouse/clickhouse-go/v2"
"github.com/google/uuid"
"github.com/shopspring/decimal"
"github.com/gogf/gf/v2/database/gdb"
"github.com/gogf/gf/v2/errors/gcode"
"github.com/gogf/gf/v2/errors/gerror"
@ -27,8 +30,6 @@ import (
"github.com/gogf/gf/v2/util/gconv"
"github.com/gogf/gf/v2/util/gtag"
"github.com/gogf/gf/v2/util/gutil"
"github.com/google/uuid"
"github.com/shopspring/decimal"
)
// Driver is the driver for postgresql database.
@ -50,7 +51,6 @@ const (
filterTypePattern = `(?i)^UPDATE|DELETE`
replaceSchemaPattern = `@(.+?)/([\w\.\-]+)+`
needParsedSqlInCtx gctx.StrKey = "NeedParsedSql"
OrmTagForStruct = gtag.ORM
driverName = "clickhouse"
)
@ -298,13 +298,20 @@ func (d *Driver) DoInsert(
keysStr = charL + strings.Join(keys, charR+","+charL) + charR
holderStr = strings.Join(valueHolder, ",")
tx gdb.TX
stdSqlResult sql.Result
stmt *gdb.Stmt
)
tx, err = d.Core.Begin(ctx)
if err != nil {
return
}
// It here uses defer to guarantee transaction be committed or roll-backed.
defer func() {
if err == nil {
_ = tx.Commit()
} else {
_ = tx.Rollback()
}
}()
stmt, err = tx.Prepare(fmt.Sprintf(
"INSERT INTO %s(%s) VALUES (%s)",
d.QuotePrefixTableName(table), keysStr,
@ -314,22 +321,23 @@ func (d *Driver) DoInsert(
return
}
for i := 0; i < len(list); i++ {
params := make([]interface{}, 0) // Values that will be committed to underlying database driver.
// Values that will be committed to underlying database driver.
params := make([]interface{}, 0)
for _, k := range keys {
params = append(params, list[i][k])
}
// Prepare is allowed to execute only once in a transaction opened by clickhouse
stdSqlResult, err = stmt.ExecContext(ctx, params...)
result, err = stmt.ExecContext(ctx, params...)
if err != nil {
return stdSqlResult, err
return
}
}
return stdSqlResult, tx.Commit()
return
}
// ConvertDataForRecord converting for any data that will be inserted into table/collection as a record.
func (d *Driver) ConvertDataForRecord(ctx context.Context, value interface{}) (map[string]interface{}, error) {
m := gconv.Map(value, OrmTagForStruct)
m := gconv.Map(value, gtag.ORM)
// transforms a value of a particular type
for k, v := range m {

29
contrib/drivers/clickhouse/clickhouse_test.go
View File

@ -263,6 +263,35 @@ func TestDriverClickhouse_InsertOne(t *testing.T) {
gtest.AssertNil(err)
}
func TestDriverClickhouse_InsertOneAutoDateTimeWrite(t *testing.T) {
connect, err := gdb.New(gdb.ConfigNode{
Host: "127.0.0.1",
Port: "9000",
User: "default",
Name: "default",
Type: "clickhouse",
Debug: false,
CreatedAt: "created",
})
gtest.AssertNil(err)
gtest.AssertNE(connect, nil)
gtest.AssertEQ(createClickhouseTableVisits(connect), nil)
defer dropClickhouseTableVisits(connect)
beforeInsertTime := time.Now()
_, err = connect.Model("visits").Data(g.Map{
"duration": float64(grand.Intn(999)),
"url": gconv.String(grand.Intn(999)),
}).Insert()
gtest.AssertNil(err)
// Query the inserted data to get the time field value
data, err := connect.Model("visits").One()
gtest.AssertNil(err)
// Get the time value from the inserted data
createdTime := data["created"].Time()
// Assert the time field value is equal to or after the beforeInsertTime
gtest.AssertGE(createdTime.Unix(), beforeInsertTime.Unix())
}
func TestDriverClickhouse_InsertMany(t *testing.T) {
connect := clickhouseConfigDB()
gtest.AssertEQ(createClickhouseTableVisits(connect), nil)

33
contrib/drivers/dm/dm.go
View File

@ -15,15 +15,19 @@ import (
"reflect"
"strconv"
"strings"
"time"
_ "gitee.com/chunanyong/dm"
"github.com/gogf/gf/v2/database/gdb"
"github.com/gogf/gf/v2/errors/gcode"
"github.com/gogf/gf/v2/errors/gerror"
"github.com/gogf/gf/v2/frame/g"
"github.com/gogf/gf/v2/os/gtime"
"github.com/gogf/gf/v2/text/gregex"
"github.com/gogf/gf/v2/text/gstr"
"github.com/gogf/gf/v2/util/gconv"
"github.com/gogf/gf/v2/util/gtag"
"github.com/gogf/gf/v2/util/gutil"
)
@ -169,6 +173,35 @@ func (d *Driver) TableFields(
return fields, nil
}
// ConvertDataForRecord converting for any data that will be inserted into table/collection as a record.
func (d *Driver) ConvertDataForRecord(ctx context.Context, value interface{}) (map[string]interface{}, error) {
m := gconv.Map(value, gtag.ORM)
// transforms a value of a particular type
for k, v := range m {
switch itemValue := v.(type) {
// dm does not support time.Time, it so here converts it to time string that it supports.
case time.Time:
m[k] = gtime.New(itemValue).String()
// If the time is zero, it then updates it to nil,
// which will insert/update the value to database as "null".
if itemValue.IsZero() {
m[k] = nil
}
// dm does not support time.Time, it so here converts it to time string that it supports.
case *time.Time:
m[k] = gtime.New(itemValue).String()
// If the time is zero, it then updates it to nil,
// which will insert/update the value to database as "null".
if itemValue == nil || itemValue.IsZero() {
m[k] = nil
}
}
}
return m, nil
}
// DoFilter deals with the sql string before commits it to underlying sql driver.
func (d *Driver) DoFilter(ctx context.Context, link gdb.Link, sql string, args []interface{}) (newSql string, newArgs []interface{}, err error) {
defer func() {

7
database/gdb/gdb_core.go
View File

@ -684,7 +684,12 @@ func (c *Core) DoUpdate(ctx context.Context, link Link, table string, data inter
return nil, err
}
}
return c.db.DoExec(ctx, link, fmt.Sprintf("UPDATE %s SET %s%s", table, updates, condition), args...)
return c.db.DoExec(ctx, link, fmt.Sprintf(
"UPDATE %s SET %s%s",
table, updates, condition,
),
args...,
)
}
// Delete does "DELETE FROM ... " statement for the table.

23
database/gdb/gdb_driver_wrapper_db.go
View File

@ -89,3 +89,26 @@ func (d *DriverWrapperDB) TableFields(
}
return
}
// DoInsert inserts or updates data forF given table.
// This function is usually used for custom interface definition, you do not need call it manually.
// The parameter `data` can be type of map/gmap/struct/*struct/[]map/[]struct, etc.
// Eg:
// Data(g.Map{"uid": 10000, "name":"john"})
// Data(g.Slice{g.Map{"uid": 10000, "name":"john"}, g.Map{"uid": 20000, "name":"smith"})
//
// The parameter `option` values are as follows:
// 0: insert: just insert, if there's unique/primary key in the data, it returns error;
// 1: replace: if there's unique/primary key in the data, it deletes it from table and inserts a new one;
// 2: save: if there's unique/primary key in the data, it updates it or else inserts a new one;
// 3: ignore: if there's unique/primary key in the data, it ignores the inserting;
func (d *DriverWrapperDB) DoInsert(ctx context.Context, link Link, table string, list List, option DoInsertOption) (result sql.Result, err error) {
// Convert data type before commit it to underlying db driver.
for i, item := range list {
list[i], err = d.DB.ConvertDataForRecord(ctx, item)
if err != nil {
return nil, err
}
}
return d.DB.DoInsert(ctx, link, table, list, option)
}

4
database/gdb/gdb_func.go
View File

@ -209,6 +209,10 @@ func GetInsertOperationByOption(option InsertOption) string {
return operator
}
func anyValueToMapBeforeToRecord(value interface{}) map[string]interface{} {
return gconv.Map(value, structTagPriority...)
}
// DataToMapDeep converts `value` to map type recursively(if attribute struct is embedded).
// The parameter `value` should be type of *map/map/*struct/struct.
// It supports embedded struct definition for struct.

59
database/gdb/gdb_model_insert.go
View File

@ -39,11 +39,7 @@ func (m *Model) Batch(batch int) *Model {
// Data(g.Map{"uid": 10000, "name":"john"})
// Data(g.Slice{g.Map{"uid": 10000, "name":"john"}, g.Map{"uid": 20000, "name":"smith"}).
func (m *Model) Data(data ...interface{}) *Model {
var (
err error
ctx = m.GetCtx()
model = m.getModel()
)
var model = m.getModel()
if len(data) > 1 {
if s := gconv.String(data[0]); gstr.Contains(s, "?") {
model.data = s
@ -88,10 +84,7 @@ func (m *Model) Data(data ...interface{}) *Model {
}
list := make(List, reflectInfo.OriginValue.Len())
for i := 0; i < reflectInfo.OriginValue.Len(); i++ {
list[i], err = m.db.ConvertDataForRecord(ctx, reflectInfo.OriginValue.Index(i).Interface())
if err != nil {
panic(err)
}
list[i] = anyValueToMapBeforeToRecord(reflectInfo.OriginValue.Index(i).Interface())
}
model.data = list
@ -108,24 +101,15 @@ func (m *Model) Data(data ...interface{}) *Model {
list = make(List, len(array))
)
for i := 0; i < len(array); i++ {
list[i], err = m.db.ConvertDataForRecord(ctx, array[i])
if err != nil {
panic(err)
}
list[i] = anyValueToMapBeforeToRecord(array[i])
}
model.data = list
} else {
model.data, err = m.db.ConvertDataForRecord(ctx, data[0])
if err != nil {
panic(err)
}
model.data = anyValueToMapBeforeToRecord(data[0])
}
case reflect.Map:
model.data, err = m.db.ConvertDataForRecord(ctx, data[0])
if err != nil {
panic(err)
}
model.data = anyValueToMapBeforeToRecord(data[0])
default:
model.data = data[0]
@ -278,53 +262,34 @@ func (m *Model) doInsertWithOption(ctx context.Context, insertOption InsertOptio
case List:
list = value
for i, v := range list {
list[i], err = m.db.ConvertDataForRecord(ctx, v)
if err != nil {
return nil, err
}
}
case Map:
var listItem map[string]interface{}
if listItem, err = m.db.ConvertDataForRecord(ctx, value); err != nil {
return nil, err
}
list = List{listItem}
list = List{value}
default:
// It uses gconv.Map here to simply fo the type converting from interface{} to map[string]interface{},
// as there's another DataToMapDeep in next logic to do the deep converting.
reflectInfo := reflection.OriginValueAndKind(newData)
switch reflectInfo.OriginKind {
// If it's slice type, it then converts it to List type.
case reflect.Slice, reflect.Array:
list = make(List, reflectInfo.OriginValue.Len())
for i := 0; i < reflectInfo.OriginValue.Len(); i++ {
list[i], err = m.db.ConvertDataForRecord(ctx, reflectInfo.OriginValue.Index(i).Interface())
list[i] = anyValueToMapBeforeToRecord(reflectInfo.OriginValue.Index(i).Interface())
}
case reflect.Map:
var listItem map[string]interface{}
if listItem, err = m.db.ConvertDataForRecord(ctx, value); err != nil {
return nil, err
}
list = List{listItem}
list = List{anyValueToMapBeforeToRecord(value)}
case reflect.Struct:
if v, ok := value.(iInterfaces); ok {
array := v.Interfaces()
list = make(List, len(array))
for i := 0; i < len(array); i++ {
list[i], err = m.db.ConvertDataForRecord(ctx, array[i])
if err != nil {
return nil, err
}
list[i] = anyValueToMapBeforeToRecord(array[i])
}
} else {
var listItem map[string]interface{}
if listItem, err = m.db.ConvertDataForRecord(ctx, value); err != nil {
return nil, err
}
list = List{listItem}
list = List{anyValueToMapBeforeToRecord(value)}
}
default:

9
database/gdb/gdb_model_update.go
View File

@ -9,9 +9,10 @@ package gdb
import (
"database/sql"
"fmt"
"github.com/gogf/gf/v2/internal/intlog"
"reflect"
"github.com/gogf/gf/v2/internal/intlog"
"github.com/gogf/gf/v2/errors/gcode"
"github.com/gogf/gf/v2/errors/gerror"
"github.com/gogf/gf/v2/internal/reflection"
@ -57,11 +58,7 @@ func (m *Model) Update(dataAndWhere ...interface{}) (result sql.Result, err erro
switch reflectInfo.OriginKind {
case reflect.Map, reflect.Struct:
var dataMap map[string]interface{}
dataMap, err = m.db.ConvertDataForRecord(ctx, m.data)
if err != nil {
return nil, err
}
var dataMap = anyValueToMapBeforeToRecord(m.data)
// Automatically update the record updating time.
if fieldNameUpdate != "" {
dataMap[fieldNameUpdate] = gtime.Now()