mirror of
https://gitee.com/johng/gf
synced 2026-06-07 10:22:11 +08:00
This pull request introduces significant improvements to the handling of the `Replace` and `Save` operations for multiple database drivers, especially for MSSQL and PostgreSQL. The changes ensure that these operations now auto-detect primary keys when conflict columns are not explicitly provided, improving usability and aligning behavior across drivers. Additionally, the pull request updates related tests to reflect these enhancements and includes some minor documentation and code cleanup. **Key changes:** ### Enhanced Replace/Save Logic for Database Drivers * **MSSQL Driver:** - `Replace` and `Save` operations now auto-detect primary keys if `OnConflict` is not specified, using the `MERGE` statement for upsert functionality. If no primary key is found in the data, a detailed error is returned. [[1]](diffhunk://#diff-87815aa559a927e2de09bd05148f9841dfc06a1b5f3ecc5e3d5fcb80323a87f8L23-R61) [[2]](diffhunk://#diff-87815aa559a927e2de09bd05148f9841dfc06a1b5f3ecc5e3d5fcb80323a87f8L43-L59) - Updated tests to verify that `Replace` correctly updates or inserts records, and that missing conflict columns are properly handled. [[1]](diffhunk://#diff-bdbde9d7d6ee14c795343767b414740c4396f4dd3e97788b1f9d4e615405a42dL141-R151) [[2]](diffhunk://#diff-26338e93e473300b1313936eb0f6826546473793442f24715fa294b595f7a805L2661-R2707) * **PostgreSQL Driver:** - Similar to MSSQL, `Replace` and `Save` now auto-detect primary keys for conflict resolution if `OnConflict` is not set, and treat `Replace` as a `Save` operation. - Adjusted tests to ensure `Save` and `Replace` work as expected, including verifying data replacement and insertion. [[1]](diffhunk://#diff-c22703c37ebb6836c332f7cd2ada570577ba4564fe39886db02f7c2d0e7a2048L93-R93) [[2]](diffhunk://#diff-c22703c37ebb6836c332f7cd2ada570577ba4564fe39886db02f7c2d0e7a2048R102) [[3]](diffhunk://#diff-c22703c37ebb6836c332f7cd2ada570577ba4564fe39886db02f7c2d0e7a2048L110-R130) * **DM Driver:** - Improved conflict detection: now checks that at least one primary key exists in the provided data when `OnConflict` is not specified, and provides clearer error messages. - Refactored to use the core method for primary key detection and removed redundant code. ### Minor Improvements and Documentation * Added clarifying comments to `DoInsert` methods for ClickHouse, DM, MSSQL, Oracle, and PostgreSQL drivers, specifying that the input list must have at least one validated record. [[1]](diffhunk://#diff-f2e003895041ed3c52b91bb8c270696adc3528d77c39d2f7137af3396267444cR19) [[2]](diffhunk://#diff-f51b30e3f0b0f1284b905385a89992efd0de2fe9ff8c5a4062344dfab17d428eR23) [[3]](diffhunk://#diff-87815aa559a927e2de09bd05148f9841dfc06a1b5f3ecc5e3d5fcb80323a87f8L23-R61) [[4]](diffhunk://#diff-f61dac3fcfd5df4a3936cd8743499c8c0fc45f4f5d0f5398ed84a0cb1603202cR24) [[5]](diffhunk://#diff-c1dfed79aaa3a432057d2bd74d270e4b4094ebcf72984f1161d4972bea009410R16-R72) * Minor code and comment cleanups, including improved formatting and error handling. [[1]](diffhunk://#diff-f61dac3fcfd5df4a3936cd8743499c8c0fc45f4f5d0f5398ed84a0cb1603202cR37) [[2]](diffhunk://#diff-f61dac3fcfd5df4a3936cd8743499c8c0fc45f4f5d0f5398ed84a0cb1603202cL96-R98) [[3]](diffhunk://#diff-f61dac3fcfd5df4a3936cd8743499c8c0fc45f4f5d0f5398ed84a0cb1603202cL106-L116) [[4]](diffhunk://#diff-a17b44c76aaac53d1f164a2bb9440a5531659f4355e7ccfabdadff8dc8633c09L170-R171) [[5]](diffhunk://#diff-56189fa9ae1df51716b50d34d7fe56bfe67a330e8ac2c6b0de7b958db6817ed5R83-R98) ### Workflow and Documentation Updates * Updated example Docker commands in the CI workflow for consistency and clarity. [[1]](diffhunk://#diff-a1a3cb9bdeb5541d148091d973cf266aa3b317e6415a86630e816cbe27cf8b9cL57-R57) [[2]](diffhunk://#diff-a1a3cb9bdeb5541d148091d973cf266aa3b317e6415a86630e816cbe27cf8b9cL78-R78) [[3]](diffhunk://#diff-a1a3cb9bdeb5541d148091d973cf266aa3b317e6415a86630e816cbe27cf8b9cL92-R92) [[4]](diffhunk://#diff-a1a3cb9bdeb5541d148091d973cf266aa3b317e6415a86630e816cbe27cf8b9cL106-R106) [[5]](diffhunk://#diff-a1a3cb9bdeb5541d148091d973cf266aa3b317e6415a86630e816cbe27cf8b9cL153-R153) [[6]](diffhunk://#diff-a1a3cb9bdeb5541d148091d973cf266aa3b317e6415a86630e816cbe27cf8b9cL164-R164) * Removed outdated note about `Replace` support from the SQLite driver documentation. These changes improve the consistency, reliability, and developer experience when performing upsert operations across different database backends. --------- Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Co-authored-by: Lance Add <1196661499@qq.com>
203 lines
6.8 KiB
Go
203 lines
6.8 KiB
Go
// Copyright GoFrame Author(https://goframe.org). All Rights Reserved.
|
|
//
|
|
// This Source Code Form is subject to the terms of the MIT License.
|
|
// If a copy of the MIT was not distributed with this file,
|
|
// You can obtain one at https://github.com/gogf/gf.
|
|
|
|
package dm
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
"fmt"
|
|
"strings"
|
|
|
|
"github.com/gogf/gf/v2/container/gset"
|
|
"github.com/gogf/gf/v2/database/gdb"
|
|
"github.com/gogf/gf/v2/errors/gcode"
|
|
"github.com/gogf/gf/v2/errors/gerror"
|
|
"github.com/gogf/gf/v2/text/gstr"
|
|
)
|
|
|
|
// DoInsert inserts or updates data for given table.
|
|
// The list parameter must contain at least one record, which was previously validated.
|
|
func (d *Driver) DoInsert(
|
|
ctx context.Context, link gdb.Link, table string, list gdb.List, option gdb.DoInsertOption,
|
|
) (result sql.Result, err error) {
|
|
switch option.InsertOption {
|
|
case gdb.InsertOptionSave:
|
|
return d.doSave(ctx, link, table, list, option)
|
|
|
|
case gdb.InsertOptionReplace:
|
|
// dm does not support REPLACE INTO syntax, use SAVE instead.
|
|
return d.doSave(ctx, link, table, list, option)
|
|
|
|
case gdb.InsertOptionIgnore:
|
|
// dm does not support INSERT IGNORE syntax, use MERGE instead.
|
|
return d.doInsertIgnore(ctx, link, table, list, option)
|
|
|
|
default:
|
|
// DM database supports IDENTITY auto-increment columns natively.
|
|
// The driver automatically returns LastInsertId through sql.Result.
|
|
//
|
|
// Note: DM IDENTITY columns cannot accept explicit ID values unless
|
|
// IDENTITY_INSERT is enabled. When using tables with IDENTITY columns,
|
|
// avoid providing explicit ID values in the data.
|
|
return d.Core.DoInsert(ctx, link, table, list, option)
|
|
}
|
|
}
|
|
|
|
// doSave support upsert for dm
|
|
func (d *Driver) doSave(ctx context.Context,
|
|
link gdb.Link, table string, list gdb.List, option gdb.DoInsertOption,
|
|
) (result sql.Result, err error) {
|
|
return d.doMergeInsert(ctx, link, table, list, option, true)
|
|
}
|
|
|
|
// doInsertIgnore implements INSERT IGNORE operation using MERGE statement for DM database.
|
|
// It only inserts records when there's no conflict on primary/unique keys.
|
|
func (d *Driver) doInsertIgnore(ctx context.Context,
|
|
link gdb.Link, table string, list gdb.List, option gdb.DoInsertOption,
|
|
) (result sql.Result, err error) {
|
|
return d.doMergeInsert(ctx, link, table, list, option, false)
|
|
}
|
|
|
|
// doMergeInsert implements MERGE-based insert operations for DM database.
|
|
// When withUpdate is true, it performs upsert (insert or update).
|
|
// When withUpdate is false, it performs insert ignore (insert only when no conflict).
|
|
func (d *Driver) doMergeInsert(
|
|
ctx context.Context,
|
|
link gdb.Link, table string, list gdb.List, option gdb.DoInsertOption, withUpdate bool,
|
|
) (result sql.Result, err error) {
|
|
// If OnConflict is not specified, automatically get the primary key of the table
|
|
conflictKeys := option.OnConflict
|
|
if len(conflictKeys) == 0 {
|
|
primaryKeys, err := d.Core.GetPrimaryKeys(ctx, table)
|
|
if err != nil {
|
|
return nil, gerror.WrapCode(
|
|
gcode.CodeInternalError,
|
|
err,
|
|
`failed to get primary keys for table`,
|
|
)
|
|
}
|
|
foundPrimaryKey := false
|
|
for _, primaryKey := range primaryKeys {
|
|
for dataKey := range list[0] {
|
|
if strings.EqualFold(dataKey, primaryKey) {
|
|
foundPrimaryKey = true
|
|
break
|
|
}
|
|
}
|
|
if foundPrimaryKey {
|
|
break
|
|
}
|
|
}
|
|
if !foundPrimaryKey {
|
|
return nil, gerror.NewCodef(
|
|
gcode.CodeMissingParameter,
|
|
`Replace/Save/InsertIgnore operation requires conflict detection: `+
|
|
`either specify OnConflict() columns or ensure table '%s' has a primary key in the data`,
|
|
table,
|
|
)
|
|
}
|
|
// TODO consider composite primary keys.
|
|
conflictKeys = primaryKeys
|
|
}
|
|
|
|
var (
|
|
one = list[0]
|
|
oneLen = len(one)
|
|
charL, charR = d.GetChars()
|
|
conflictKeySet = gset.New(false)
|
|
|
|
// queryHolders: Handle data with Holder that need to be merged
|
|
// queryValues: Handle data that need to be merged
|
|
// insertKeys: Handle valid keys that need to be inserted
|
|
// insertValues: Handle values that need to be inserted
|
|
// updateValues: Handle values that need to be updated (only when withUpdate=true)
|
|
queryHolders = make([]string, oneLen)
|
|
queryValues = make([]any, oneLen)
|
|
insertKeys = make([]string, oneLen)
|
|
insertValues = make([]string, oneLen)
|
|
updateValues []string
|
|
)
|
|
|
|
// conflictKeys slice type conv to set type
|
|
for _, conflictKey := range conflictKeys {
|
|
conflictKeySet.Add(gstr.ToUpper(conflictKey))
|
|
}
|
|
|
|
index := 0
|
|
for key, value := range one {
|
|
keyWithChar := charL + key + charR
|
|
queryHolders[index] = fmt.Sprintf("? AS %s", keyWithChar)
|
|
queryValues[index] = value
|
|
insertKeys[index] = keyWithChar
|
|
insertValues[index] = fmt.Sprintf("T2.%s", keyWithChar)
|
|
|
|
// Build updateValues only when withUpdate is true
|
|
// Filter conflict keys and soft created fields from updateValues
|
|
if withUpdate && !(conflictKeySet.Contains(key) || d.Core.IsSoftCreatedFieldName(key)) {
|
|
updateValues = append(
|
|
updateValues,
|
|
fmt.Sprintf(`T1.%s = T2.%s`, keyWithChar, keyWithChar),
|
|
)
|
|
}
|
|
index++
|
|
}
|
|
|
|
var (
|
|
batchResult = new(gdb.SqlResult)
|
|
sqlStr = parseSqlForMerge(table, queryHolders, insertKeys, insertValues, updateValues, conflictKeys)
|
|
)
|
|
r, err := d.DoExec(ctx, link, sqlStr, queryValues...)
|
|
if err != nil {
|
|
return r, err
|
|
}
|
|
if n, err := r.RowsAffected(); err != nil {
|
|
return r, err
|
|
} else {
|
|
batchResult.Result = r
|
|
batchResult.Affected += n
|
|
}
|
|
return batchResult, nil
|
|
}
|
|
|
|
// parseSqlForMerge generates MERGE statement for DM database.
|
|
// When updateValues is empty, it only inserts (INSERT IGNORE behavior).
|
|
// When updateValues is provided, it performs upsert (INSERT or UPDATE).
|
|
// Examples:
|
|
// - INSERT IGNORE: MERGE INTO table T1 USING (...) T2 ON (...) WHEN NOT MATCHED THEN INSERT(...) VALUES (...)
|
|
// - UPSERT: MERGE INTO table T1 USING (...) T2 ON (...) WHEN NOT MATCHED THEN INSERT(...) VALUES (...) WHEN MATCHED THEN UPDATE SET ...
|
|
func parseSqlForMerge(table string,
|
|
queryHolders, insertKeys, insertValues, updateValues, duplicateKey []string,
|
|
) (sqlStr string) {
|
|
var (
|
|
queryHolderStr = strings.Join(queryHolders, ",")
|
|
insertKeyStr = strings.Join(insertKeys, ",")
|
|
insertValueStr = strings.Join(insertValues, ",")
|
|
duplicateKeyStr string
|
|
)
|
|
|
|
// Build ON condition
|
|
for index, keys := range duplicateKey {
|
|
if index != 0 {
|
|
duplicateKeyStr += " AND "
|
|
}
|
|
duplicateKeyStr += fmt.Sprintf("T1.%s = T2.%s", keys, keys)
|
|
}
|
|
|
|
// Build SQL based on whether UPDATE is needed
|
|
pattern := gstr.Trim(`MERGE INTO %s T1 USING (SELECT %s FROM DUAL) T2 ON (%s) WHEN NOT MATCHED THEN INSERT(%s) VALUES (%s)`)
|
|
if len(updateValues) > 0 {
|
|
// Upsert: INSERT or UPDATE
|
|
pattern += gstr.Trim(`WHEN MATCHED THEN UPDATE SET %s`)
|
|
return fmt.Sprintf(
|
|
pattern, table, queryHolderStr, duplicateKeyStr, insertKeyStr, insertValueStr,
|
|
strings.Join(updateValues, ","),
|
|
)
|
|
}
|
|
// Insert Ignore: INSERT only
|
|
return fmt.Sprintf(pattern, table, queryHolderStr, duplicateKeyStr, insertKeyStr, insertValueStr)
|
|
}
|