remove Batch*/DoBatchInsert functions for package gdb

This commit is contained in:
John Guo
2021-06-08 20:32:34 +08:00
parent 3ac5772059
commit e68e7a3224
9 changed files with 73 additions and 347 deletions

View File

@ -83,16 +83,11 @@ type DB interface {
// Common APIs for CURD.
// ===========================================================================
Insert(table string, data interface{}, batch ...int) (sql.Result, error) // See Core.Insert.
InsertIgnore(table string, data interface{}, batch ...int) (sql.Result, error) // See Core.InsertIgnore.
InsertAndGetId(table string, data interface{}, batch ...int) (int64, error) // See Core.InsertAndGetId.
Replace(table string, data interface{}, batch ...int) (sql.Result, error) // See Core.Replace.
Save(table string, data interface{}, batch ...int) (sql.Result, error) // See Core.Save.
BatchInsert(table string, list interface{}, batch ...int) (sql.Result, error) // See Core.BatchInsert.
BatchReplace(table string, list interface{}, batch ...int) (sql.Result, error) // See Core.BatchReplace.
BatchSave(table string, list interface{}, batch ...int) (sql.Result, error) // See Core.BatchSave.
Insert(table string, data interface{}, batch ...int) (sql.Result, error) // See Core.Insert.
InsertIgnore(table string, data interface{}, batch ...int) (sql.Result, error) // See Core.InsertIgnore.
InsertAndGetId(table string, data interface{}, batch ...int) (int64, error) // See Core.InsertAndGetId.
Replace(table string, data interface{}, batch ...int) (sql.Result, error) // See Core.Replace.
Save(table string, data interface{}, batch ...int) (sql.Result, error) // See Core.Save.
Update(table string, data interface{}, condition interface{}, args ...interface{}) (sql.Result, error) // See Core.Update.
Delete(table string, condition interface{}, args ...interface{}) (sql.Result, error) // See Core.Delete.
@ -104,8 +99,7 @@ type DB interface {
DoExec(ctx context.Context, link Link, sql string, args ...interface{}) (result sql.Result, err error) // See Core.DoExec.
DoPrepare(ctx context.Context, link Link, sql string) (*Stmt, error) // See Core.DoPrepare.
DoGetAll(ctx context.Context, link Link, sql string, args ...interface{}) (result Result, err error) // See Core.DoGetAll.
DoInsert(ctx context.Context, link Link, table string, data interface{}, option int, batch ...int) (result sql.Result, err error) // See Core.DoInsert.
DoBatchInsert(ctx context.Context, link Link, table string, list interface{}, option int, batch ...int) (result sql.Result, err error) // See Core.DoBatchInsert.
DoInsert(ctx context.Context, link Link, table string, data interface{}, option int, batch int) (result sql.Result, err error) // See Core.DoInsert.
DoUpdate(ctx context.Context, link Link, table string, data interface{}, condition string, args ...interface{}) (result sql.Result, err error) // See Core.DoUpdate.
DoDelete(ctx context.Context, link Link, table string, condition string, args ...interface{}) (result sql.Result, err error) // See Core.DoDelete.

View File

@ -367,120 +367,7 @@ func (c *Core) Save(table string, data interface{}, batch ...int) (sql.Result, e
// 1: replace: if there's unique/primary key in the data, it deletes it from table and inserts a new one;
// 2: save: if there's unique/primary key in the data, it updates it or else inserts a new one;
// 3: ignore: if there's unique/primary key in the data, it ignores the inserting;
func (c *Core) DoInsert(ctx context.Context, link Link, table string, data interface{}, option int, batch ...int) (result sql.Result, err error) {
table = c.QuotePrefixTableName(table)
var (
fields []string
values []string
params []interface{}
dataMap Map
reflectValue = reflect.ValueOf(data)
reflectKind = reflectValue.Kind()
)
if reflectKind == reflect.Ptr {
reflectValue = reflectValue.Elem()
reflectKind = reflectValue.Kind()
}
switch reflectKind {
case reflect.Slice, reflect.Array:
return c.db.DoBatchInsert(ctx, link, table, data, option, batch...)
case reflect.Struct:
if _, ok := data.(apiInterfaces); ok {
return c.db.DoBatchInsert(ctx, link, table, data, option, batch...)
} else {
dataMap = ConvertDataForTableRecord(data)
}
case reflect.Map:
dataMap = ConvertDataForTableRecord(data)
default:
return result, gerror.New(fmt.Sprint("unsupported data type:", reflectKind))
}
if len(dataMap) == 0 {
return nil, gerror.New("data cannot be empty")
}
var (
charL, charR = c.db.GetChars()
operation = GetInsertOperationByOption(option)
updateStr = ""
)
for k, v := range dataMap {
fields = append(fields, charL+k+charR)
if s, ok := v.(Raw); ok {
values = append(values, gconv.String(s))
} else {
values = append(values, "?")
params = append(params, v)
}
}
if option == insertOptionSave {
for k, _ := range dataMap {
// If it's SAVE operation,
// do not automatically update the creating time.
if c.isSoftCreatedFiledName(k) {
continue
}
if len(updateStr) > 0 {
updateStr += ","
}
updateStr += fmt.Sprintf(
"%s%s%s=VALUES(%s%s%s)",
charL, k, charR,
charL, k, charR,
)
}
updateStr = fmt.Sprintf("ON DUPLICATE KEY UPDATE %s", updateStr)
}
if link == nil {
if link, err = c.MasterLink(); err != nil {
return nil, err
}
}
return c.db.DoExec(ctx, link, fmt.Sprintf(
"%s INTO %s(%s) VALUES(%s) %s",
operation, table, strings.Join(fields, ","),
strings.Join(values, ","), updateStr,
), params...)
}
// BatchInsert batch inserts data.
// The parameter `list` must be type of slice of map or struct.
func (c *Core) BatchInsert(table string, list interface{}, batch ...int) (sql.Result, error) {
if len(batch) > 0 {
return c.Model(table).Data(list).Batch(batch[0]).Insert()
}
return c.Model(table).Data(list).Insert()
}
// BatchInsertIgnore batch inserts data with ignore option.
// The parameter `list` must be type of slice of map or struct.
func (c *Core) BatchInsertIgnore(table string, list interface{}, batch ...int) (sql.Result, error) {
if len(batch) > 0 {
return c.Model(table).Data(list).Batch(batch[0]).InsertIgnore()
}
return c.Model(table).Data(list).InsertIgnore()
}
// BatchReplace batch replaces data.
// The parameter `list` must be type of slice of map or struct.
func (c *Core) BatchReplace(table string, list interface{}, batch ...int) (sql.Result, error) {
if len(batch) > 0 {
return c.Model(table).Data(list).Batch(batch[0]).Replace()
}
return c.Model(table).Data(list).Replace()
}
// BatchSave batch replaces data.
// The parameter `list` must be type of slice of map or struct.
func (c *Core) BatchSave(table string, list interface{}, batch ...int) (sql.Result, error) {
if len(batch) > 0 {
return c.Model(table).Data(list).Batch(batch[0]).Save()
}
return c.Model(table).Data(list).Save()
}
// DoBatchInsert batch inserts/replaces/saves data.
// This function is usually used for custom interface definition, you do not need call it manually.
func (c *Core) DoBatchInsert(ctx context.Context, link Link, table string, list interface{}, option int, batch ...int) (result sql.Result, err error) {
func (c *Core) DoInsert(ctx context.Context, link Link, table string, data interface{}, option int, batch int) (result sql.Result, err error) {
table = c.QuotePrefixTableName(table)
var (
keys []string // Field names.
@ -488,18 +375,25 @@ func (c *Core) DoBatchInsert(ctx context.Context, link Link, table string, list
params []interface{} // Values that will be committed to underlying database driver.
listMap List // The data list that passed from caller.
)
switch value := list.(type) {
switch value := data.(type) {
case Result:
listMap = value.List()
case Record:
listMap = List{value.Map()}
case List:
listMap = value
for i, v := range listMap {
listMap[i] = ConvertDataForTableRecord(v)
}
case Map:
listMap = List{value}
listMap = List{ConvertDataForTableRecord(value)}
default:
var (
rv = reflect.ValueOf(list)
rv = reflect.ValueOf(data)
kind = rv.Kind()
)
if kind == reflect.Ptr {
@ -513,8 +407,10 @@ func (c *Core) DoBatchInsert(ctx context.Context, link Link, table string, list
for i := 0; i < rv.Len(); i++ {
listMap[i] = ConvertDataForTableRecord(rv.Index(i).Interface())
}
case reflect.Map:
listMap = List{ConvertDataForTableRecord(value)}
case reflect.Struct:
if v, ok := value.(apiInterfaces); ok {
var (
@ -528,6 +424,7 @@ func (c *Core) DoBatchInsert(ctx context.Context, link Link, table string, list
} else {
listMap = List{ConvertDataForTableRecord(value)}
}
default:
return result, gerror.New(fmt.Sprint("unsupported list type:", kind))
}
@ -570,9 +467,8 @@ func (c *Core) DoBatchInsert(ctx context.Context, link Link, table string, list
}
updateStr = fmt.Sprintf("ON DUPLICATE KEY UPDATE %s", updateStr)
}
batchNum := defaultBatchNumber
if len(batch) > 0 && batch[0] > 0 {
batchNum = batch[0]
if batch <= 0 {
batch = defaultBatchNumber
}
var (
listMapLen = len(listMap)
@ -591,7 +487,8 @@ func (c *Core) DoBatchInsert(ctx context.Context, link Link, table string, list
}
}
valueHolder = append(valueHolder, "("+gstr.Join(values, ",")+")")
if len(valueHolder) == batchNum || (i == listMapLen-1 && len(valueHolder) > 0) {
// Batch package checks: It meets the batch number or it is the last element.
if len(valueHolder) == batch || (i == listMapLen-1 && len(valueHolder) > 0) {
r, err := c.db.DoExec(ctx, link, fmt.Sprintf(
"%s INTO %s(%s) VALUES%s %s",
operation, table, keysStr,

View File

@ -503,42 +503,6 @@ func (tx *TX) Save(table string, data interface{}, batch ...int) (sql.Result, er
return tx.Model(table).Ctx(tx.ctx).Data(data).Save()
}
// BatchInsert batch inserts data.
// The parameter `list` must be type of slice of map or struct.
func (tx *TX) BatchInsert(table string, list interface{}, batch ...int) (sql.Result, error) {
if len(batch) > 0 {
return tx.Model(table).Ctx(tx.ctx).Data(list).Batch(batch[0]).Insert()
}
return tx.Model(table).Ctx(tx.ctx).Data(list).Insert()
}
// BatchInsertIgnore batch inserts data with ignore option.
// The parameter `list` must be type of slice of map or struct.
func (tx *TX) BatchInsertIgnore(table string, list interface{}, batch ...int) (sql.Result, error) {
if len(batch) > 0 {
return tx.Model(table).Ctx(tx.ctx).Data(list).Batch(batch[0]).InsertIgnore()
}
return tx.Model(table).Ctx(tx.ctx).Data(list).InsertIgnore()
}
// BatchReplace batch replaces data.
// The parameter `list` must be type of slice of map or struct.
func (tx *TX) BatchReplace(table string, list interface{}, batch ...int) (sql.Result, error) {
if len(batch) > 0 {
return tx.Model(table).Ctx(tx.ctx).Data(list).Batch(batch[0]).Replace()
}
return tx.Model(table).Ctx(tx.ctx).Data(list).Replace()
}
// BatchSave batch replaces data.
// The parameter `list` must be type of slice of map or struct.
func (tx *TX) BatchSave(table string, list interface{}, batch ...int) (sql.Result, error) {
if len(batch) > 0 {
return tx.Model(table).Ctx(tx.ctx).Data(list).Batch(batch[0]).Save()
}
return tx.Model(table).Ctx(tx.ctx).Data(list).Save()
}
// Update does "UPDATE ... " statement for the table.
//
// The parameter `data` can be type of string/map/gmap/struct/*struct, etc.

View File

@ -264,114 +264,7 @@ func (d *DriverOracle) getTableUniqueIndex(table string) (fields map[string]map[
return
}
func (d *DriverOracle) DoInsert(ctx context.Context, link Link, table string, data interface{}, option int, batch ...int) (result sql.Result, err error) {
var (
fields []string
values []string
params []interface{}
dataMap Map
rv = reflect.ValueOf(data)
kind = rv.Kind()
)
if kind == reflect.Ptr {
rv = rv.Elem()
kind = rv.Kind()
}
switch kind {
case reflect.Slice, reflect.Array:
return d.DoBatchInsert(ctx, link, table, data, option, batch...)
case reflect.Map:
fallthrough
case reflect.Struct:
dataMap = ConvertDataForTableRecord(data)
default:
return result, gerror.New(fmt.Sprint("unsupported data type:", kind))
}
var (
indexes = make([]string, 0)
indexMap = make(map[string]string)
indexExists = false
)
if option != insertOptionDefault {
index, err := d.getTableUniqueIndex(table)
if err != nil {
return nil, err
}
if len(index) > 0 {
for _, v := range index {
for k, _ := range v {
indexes = append(indexes, k)
}
indexMap = v
indexExists = true
break
}
}
}
var (
subSqlStr = make([]string, 0)
onStr = make([]string, 0)
updateStr = make([]string, 0)
)
charL, charR := d.db.GetChars()
for k, v := range dataMap {
k = strings.ToUpper(k)
// 操作类型为REPLACE/SAVE时且存在唯一索引才使用merge否则使用insert
if (option == insertOptionReplace || option == insertOptionSave) && indexExists {
fields = append(fields, tableAlias1+"."+charL+k+charR)
values = append(values, tableAlias2+"."+charL+k+charR)
params = append(params, v)
subSqlStr = append(subSqlStr, fmt.Sprintf("%s?%s %s", charL, charR, k))
//m erge中的on子句中由唯一索引组成, update子句中不含唯一索引
if _, ok := indexMap[k]; ok {
onStr = append(onStr, fmt.Sprintf("%s.%s = %s.%s ", tableAlias1, k, tableAlias2, k))
} else {
updateStr = append(updateStr, fmt.Sprintf("%s.%s = %s.%s ", tableAlias1, k, tableAlias2, k))
}
} else {
fields = append(fields, charL+k+charR)
values = append(values, "?")
params = append(params, v)
}
}
if link == nil {
if link, err = d.MasterLink(); err != nil {
return nil, err
}
}
if indexExists && option != insertOptionDefault {
switch option {
case
insertOptionReplace,
insertOptionSave:
tmp := fmt.Sprintf(
"MERGE INTO %s %s USING(SELECT %s FROM DUAL) %s ON(%s) WHEN MATCHED THEN UPDATE SET %s WHEN NOT MATCHED THEN INSERT (%s) VALUES(%s)",
table, tableAlias1, strings.Join(subSqlStr, ","), tableAlias2,
strings.Join(onStr, "AND"), strings.Join(updateStr, ","), strings.Join(fields, ","), strings.Join(values, ","),
)
return d.DoExec(ctx, link, tmp, params...)
case insertOptionIgnore:
return d.DoExec(ctx, link, fmt.Sprintf(
"INSERT /*+ IGNORE_ROW_ON_DUPKEY_INDEX(%s(%s)) */ INTO %s(%s) VALUES(%s)",
table, strings.Join(indexes, ","), table, strings.Join(fields, ","), strings.Join(values, ","),
), params...)
}
}
return d.DoExec(ctx, link,
fmt.Sprintf(
"INSERT INTO %s(%s) VALUES(%s)",
table, strings.Join(fields, ","), strings.Join(values, ","),
),
params...)
}
func (d *DriverOracle) DoBatchInsert(ctx context.Context, link Link, table string, list interface{}, option int, batch ...int) (result sql.Result, err error) {
func (d *DriverOracle) DoInsert(ctx context.Context, link Link, table string, list interface{}, option int, batch int) (result sql.Result, err error) {
var (
keys []string
values []string
@ -447,9 +340,8 @@ func (d *DriverOracle) DoBatchInsert(ctx context.Context, link Link, table strin
return batchResult, nil
}
batchNum := defaultBatchNumber
if len(batch) > 0 {
batchNum = batch[0]
if batch <= 0 {
batch = defaultBatchNumber
}
// Format "INSERT...INTO..." statement.
intoStr := make([]string, 0)
@ -459,7 +351,7 @@ func (d *DriverOracle) DoBatchInsert(ctx context.Context, link Link, table strin
}
values = append(values, valueHolderStr)
intoStr = append(intoStr, fmt.Sprintf(" INTO %s(%s) VALUES(%s) ", table, keyStr, valueHolderStr))
if len(intoStr) == batchNum {
if len(intoStr) == batch {
r, err := d.DoExec(ctx, link, fmt.Sprintf("INSERT ALL %s SELECT * FROM DUAL", strings.Join(intoStr, " ")), params...)
if err != nil {
return r, err

View File

@ -142,8 +142,8 @@ func GetInsertOperationByOption(option int) string {
// ConvertDataForTableRecord is a very important function, which does converting for any data that
// will be inserted into table as a record.
//
// The parameter `obj` should be type of *map/map/*struct/struct.
// It supports inherit struct definition for struct.
// The parameter `value` should be type of *map/map/*struct/struct.
// It supports embedded struct definition for struct.
func ConvertDataForTableRecord(value interface{}) map[string]interface{} {
var (
rvValue reflect.Value
@ -186,7 +186,7 @@ func ConvertDataForTableRecord(value interface{}) map[string]interface{} {
// DataToMapDeep converts `value` to map type recursively.
// The parameter `value` should be type of *map/map/*struct/struct.
// It supports inherit struct definition for struct.
// It supports embedded struct definition for struct.
func DataToMapDeep(value interface{}) map[string]interface{} {
if v, ok := value.(apiMapStrAny); ok {
return v.MapStrAny()

View File

@ -166,68 +166,47 @@ func (m *Model) doInsertWithOption(option int) (result sql.Result, err error) {
return nil, gerror.New("inserting into table with empty data")
}
var (
list List
nowString = gtime.Now().String()
fieldNameCreate = m.getSoftFieldNameCreated()
fieldNameUpdate = m.getSoftFieldNameUpdated()
fieldNameDelete = m.getSoftFieldNameDeleted()
)
// Batch operation.
if list, ok := m.data.(List); ok {
batch := defaultBatchNumber
if m.batch > 0 {
batch = m.batch
}
newData, err := m.filterDataForInsertOrUpdate(list)
if err != nil {
return nil, err
}
list = newData.(List)
// Automatic handling for creating/updating time.
if !m.unscoped && (fieldNameCreate != "" || fieldNameUpdate != "") {
for k, v := range list {
gutil.MapDelete(v, fieldNameCreate, fieldNameUpdate, fieldNameDelete)
if fieldNameCreate != "" {
v[fieldNameCreate] = nowString
}
if fieldNameUpdate != "" {
v[fieldNameUpdate] = nowString
}
list[k] = v
}
}
return m.db.DoBatchInsert(
m.GetCtx(),
m.getLink(true),
m.tables,
newData,
option,
batch,
)
newData, err := m.filterDataForInsertOrUpdate(m.data)
if err != nil {
return nil, err
}
// Single operation.
if data, ok := m.data.(Map); ok {
newData, err := m.filterDataForInsertOrUpdate(data)
if err != nil {
return nil, err
}
data = newData.(Map)
// Automatic handling for creating/updating time.
if !m.unscoped && (fieldNameCreate != "" || fieldNameUpdate != "") {
gutil.MapDelete(data, fieldNameCreate, fieldNameUpdate, fieldNameDelete)
// It converts any data to List type for inserting.
switch newData.(type) {
case Map:
list = List{newData.(Map)}
case List:
list = newData.(List)
default:
return nil, gerror.New("inserting into table with invalid data type")
}
// Automatic handling for creating/updating time.
if !m.unscoped && (fieldNameCreate != "" || fieldNameUpdate != "") {
for k, v := range list {
gutil.MapDelete(v, fieldNameCreate, fieldNameUpdate, fieldNameDelete)
if fieldNameCreate != "" {
data[fieldNameCreate] = nowString
v[fieldNameCreate] = nowString
}
if fieldNameUpdate != "" {
data[fieldNameUpdate] = nowString
v[fieldNameUpdate] = nowString
}
list[k] = v
}
return m.db.DoInsert(
m.GetCtx(),
m.getLink(true),
m.tables,
newData,
option,
)
}
return nil, gerror.New("inserting into table with invalid data type")
return m.db.DoInsert(m.GetCtx(), m.getLink(true), m.tables, list, option, m.getBatch())
}
func (m *Model) getBatch() int {
batch := defaultBatchNumber
if m.batch > 0 {
batch = m.batch
}
return batch
}

View File

@ -195,7 +195,7 @@ func createInitTableWithDb(db gdb.DB, table ...string) (name string) {
})
}
result, err := db.BatchInsert(name, array.Slice())
result, err := db.Insert(name, array.Slice())
gtest.AssertNil(err)
n, e := result.RowsAffected()

View File

@ -329,7 +329,7 @@ func Test_DB_BatchInsert(t *testing.T) {
gtest.C(t, func(t *gtest.T) {
table := createTable()
defer dropTable(table)
r, err := db.BatchInsert(table, g.List{
r, err := db.Insert(table, g.List{
{
"id": 2,
"passport": "t2",
@ -357,7 +357,7 @@ func Test_DB_BatchInsert(t *testing.T) {
table := createTable()
defer dropTable(table)
// []interface{}
r, err := db.BatchInsert(table, g.Slice{
r, err := db.Insert(table, g.Slice{
g.Map{
"id": 2,
"passport": "t2",
@ -382,7 +382,7 @@ func Test_DB_BatchInsert(t *testing.T) {
gtest.C(t, func(t *gtest.T) {
table := createTable()
defer dropTable(table)
result, err := db.BatchInsert(table, g.Map{
result, err := db.Insert(table, g.Map{
"id": 1,
"passport": "t1",
"password": "p1",
@ -416,7 +416,7 @@ func Test_DB_BatchInsert_Struct(t *testing.T) {
NickName: "T1",
CreateTime: gtime.Now(),
}
result, err := db.BatchInsert(table, user)
result, err := db.Insert(table, user)
t.AssertNil(err)
n, _ := result.RowsAffected()
t.Assert(n, 1)
@ -1283,7 +1283,7 @@ func Test_DB_Prefix(t *testing.T) {
})
}
result, err := db.BatchInsert(name, array.Slice())
result, err := db.Insert(name, array.Slice())
t.AssertNil(err)
n, e := result.RowsAffected()

View File

@ -163,7 +163,7 @@ func Test_TX_BatchInsert(t *testing.T) {
if err != nil {
gtest.Error(err)
}
if _, err := tx.BatchInsert(table, g.List{
if _, err := tx.Insert(table, g.List{
{
"id": 2,
"passport": "t",
@ -201,7 +201,7 @@ func Test_TX_BatchReplace(t *testing.T) {
if err != nil {
gtest.Error(err)
}
if _, err := tx.BatchReplace(table, g.List{
if _, err := tx.Replace(table, g.List{
{
"id": 2,
"passport": "USER_2",
@ -244,7 +244,7 @@ func Test_TX_BatchSave(t *testing.T) {
if err != nil {
gtest.Error(err)
}
if _, err := tx.BatchSave(table, g.List{
if _, err := tx.Save(table, g.List{
{
"id": 4,
"passport": "USER_4",
@ -956,8 +956,8 @@ func Test_Transaction_Nested_TX_Transaction_UseDB(t *testing.T) {
table := createTable()
defer dropTable(table)
db.SetDebug(true)
defer db.SetDebug(false)
//db.SetDebug(true)
//defer db.SetDebug(false)
gtest.C(t, func(t *gtest.T) {
var (