Files
gf/contrib/drivers/dm/dm_do_insert.go
John Guo 852c3dda62 feat(contrib/drivers/dm&pgsql&mssql&oracle): add Replace/LastInsertId features support for dm/pgsql/mssql/oracle (#4547)
This pull request introduces significant improvements to the handling of
the `Replace` and `Save` operations for multiple database drivers,
especially for MSSQL and PostgreSQL. The changes ensure that these
operations now auto-detect primary keys when conflict columns are not
explicitly provided, improving usability and aligning behavior across
drivers. Additionally, the pull request updates related tests to reflect
these enhancements and includes some minor documentation and code
cleanup.

**Key changes:**

### Enhanced Replace/Save Logic for Database Drivers

* **MSSQL Driver:**
- `Replace` and `Save` operations now auto-detect primary keys if
`OnConflict` is not specified, using the `MERGE` statement for upsert
functionality. If no primary key is found in the data, a detailed error
is returned.
[[1]](diffhunk://#diff-87815aa559a927e2de09bd05148f9841dfc06a1b5f3ecc5e3d5fcb80323a87f8L23-R61)
[[2]](diffhunk://#diff-87815aa559a927e2de09bd05148f9841dfc06a1b5f3ecc5e3d5fcb80323a87f8L43-L59)
- Updated tests to verify that `Replace` correctly updates or inserts
records, and that missing conflict columns are properly handled.
[[1]](diffhunk://#diff-bdbde9d7d6ee14c795343767b414740c4396f4dd3e97788b1f9d4e615405a42dL141-R151)
[[2]](diffhunk://#diff-26338e93e473300b1313936eb0f6826546473793442f24715fa294b595f7a805L2661-R2707)

* **PostgreSQL Driver:**
- Similar to MSSQL, `Replace` and `Save` now auto-detect primary keys
for conflict resolution if `OnConflict` is not set, and treat `Replace`
as a `Save` operation.
- Adjusted tests to ensure `Save` and `Replace` work as expected,
including verifying data replacement and insertion.
[[1]](diffhunk://#diff-c22703c37ebb6836c332f7cd2ada570577ba4564fe39886db02f7c2d0e7a2048L93-R93)
[[2]](diffhunk://#diff-c22703c37ebb6836c332f7cd2ada570577ba4564fe39886db02f7c2d0e7a2048R102)
[[3]](diffhunk://#diff-c22703c37ebb6836c332f7cd2ada570577ba4564fe39886db02f7c2d0e7a2048L110-R130)

* **DM Driver:**
- Improved conflict detection: now checks that at least one primary key
exists in the provided data when `OnConflict` is not specified, and
provides clearer error messages.
- Refactored to use the core method for primary key detection and
removed redundant code.

### Minor Improvements and Documentation

* Added clarifying comments to `DoInsert` methods for ClickHouse, DM,
MSSQL, Oracle, and PostgreSQL drivers, specifying that the input list
must have at least one validated record.
[[1]](diffhunk://#diff-f2e003895041ed3c52b91bb8c270696adc3528d77c39d2f7137af3396267444cR19)
[[2]](diffhunk://#diff-f51b30e3f0b0f1284b905385a89992efd0de2fe9ff8c5a4062344dfab17d428eR23)
[[3]](diffhunk://#diff-87815aa559a927e2de09bd05148f9841dfc06a1b5f3ecc5e3d5fcb80323a87f8L23-R61)
[[4]](diffhunk://#diff-f61dac3fcfd5df4a3936cd8743499c8c0fc45f4f5d0f5398ed84a0cb1603202cR24)
[[5]](diffhunk://#diff-c1dfed79aaa3a432057d2bd74d270e4b4094ebcf72984f1161d4972bea009410R16-R72)
* Minor code and comment cleanups, including improved formatting and
error handling.
[[1]](diffhunk://#diff-f61dac3fcfd5df4a3936cd8743499c8c0fc45f4f5d0f5398ed84a0cb1603202cR37)
[[2]](diffhunk://#diff-f61dac3fcfd5df4a3936cd8743499c8c0fc45f4f5d0f5398ed84a0cb1603202cL96-R98)
[[3]](diffhunk://#diff-f61dac3fcfd5df4a3936cd8743499c8c0fc45f4f5d0f5398ed84a0cb1603202cL106-L116)
[[4]](diffhunk://#diff-a17b44c76aaac53d1f164a2bb9440a5531659f4355e7ccfabdadff8dc8633c09L170-R171)
[[5]](diffhunk://#diff-56189fa9ae1df51716b50d34d7fe56bfe67a330e8ac2c6b0de7b958db6817ed5R83-R98)

### Workflow and Documentation Updates

* Updated example Docker commands in the CI workflow for consistency and
clarity.
[[1]](diffhunk://#diff-a1a3cb9bdeb5541d148091d973cf266aa3b317e6415a86630e816cbe27cf8b9cL57-R57)
[[2]](diffhunk://#diff-a1a3cb9bdeb5541d148091d973cf266aa3b317e6415a86630e816cbe27cf8b9cL78-R78)
[[3]](diffhunk://#diff-a1a3cb9bdeb5541d148091d973cf266aa3b317e6415a86630e816cbe27cf8b9cL92-R92)
[[4]](diffhunk://#diff-a1a3cb9bdeb5541d148091d973cf266aa3b317e6415a86630e816cbe27cf8b9cL106-R106)
[[5]](diffhunk://#diff-a1a3cb9bdeb5541d148091d973cf266aa3b317e6415a86630e816cbe27cf8b9cL153-R153)
[[6]](diffhunk://#diff-a1a3cb9bdeb5541d148091d973cf266aa3b317e6415a86630e816cbe27cf8b9cL164-R164)
* Removed outdated note about `Replace` support from the SQLite driver
documentation.

These changes improve the consistency, reliability, and developer
experience when performing upsert operations across different database
backends.

---------

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: Lance Add <1196661499@qq.com>
2025-12-09 15:46:41 +08:00

203 lines
6.8 KiB
Go

// Copyright GoFrame Author(https://goframe.org). All Rights Reserved.
//
// This Source Code Form is subject to the terms of the MIT License.
// If a copy of the MIT was not distributed with this file,
// You can obtain one at https://github.com/gogf/gf.
package dm
import (
"context"
"database/sql"
"fmt"
"strings"
"github.com/gogf/gf/v2/container/gset"
"github.com/gogf/gf/v2/database/gdb"
"github.com/gogf/gf/v2/errors/gcode"
"github.com/gogf/gf/v2/errors/gerror"
"github.com/gogf/gf/v2/text/gstr"
)
// DoInsert inserts or updates data for given table.
// The list parameter must contain at least one record, which was previously validated.
func (d *Driver) DoInsert(
ctx context.Context, link gdb.Link, table string, list gdb.List, option gdb.DoInsertOption,
) (result sql.Result, err error) {
switch option.InsertOption {
case gdb.InsertOptionSave:
return d.doSave(ctx, link, table, list, option)
case gdb.InsertOptionReplace:
// dm does not support REPLACE INTO syntax, use SAVE instead.
return d.doSave(ctx, link, table, list, option)
case gdb.InsertOptionIgnore:
// dm does not support INSERT IGNORE syntax, use MERGE instead.
return d.doInsertIgnore(ctx, link, table, list, option)
default:
// DM database supports IDENTITY auto-increment columns natively.
// The driver automatically returns LastInsertId through sql.Result.
//
// Note: DM IDENTITY columns cannot accept explicit ID values unless
// IDENTITY_INSERT is enabled. When using tables with IDENTITY columns,
// avoid providing explicit ID values in the data.
return d.Core.DoInsert(ctx, link, table, list, option)
}
}
// doSave support upsert for dm
func (d *Driver) doSave(ctx context.Context,
link gdb.Link, table string, list gdb.List, option gdb.DoInsertOption,
) (result sql.Result, err error) {
return d.doMergeInsert(ctx, link, table, list, option, true)
}
// doInsertIgnore implements INSERT IGNORE operation using MERGE statement for DM database.
// It only inserts records when there's no conflict on primary/unique keys.
func (d *Driver) doInsertIgnore(ctx context.Context,
link gdb.Link, table string, list gdb.List, option gdb.DoInsertOption,
) (result sql.Result, err error) {
return d.doMergeInsert(ctx, link, table, list, option, false)
}
// doMergeInsert implements MERGE-based insert operations for DM database.
// When withUpdate is true, it performs upsert (insert or update).
// When withUpdate is false, it performs insert ignore (insert only when no conflict).
func (d *Driver) doMergeInsert(
ctx context.Context,
link gdb.Link, table string, list gdb.List, option gdb.DoInsertOption, withUpdate bool,
) (result sql.Result, err error) {
// If OnConflict is not specified, automatically get the primary key of the table
conflictKeys := option.OnConflict
if len(conflictKeys) == 0 {
primaryKeys, err := d.Core.GetPrimaryKeys(ctx, table)
if err != nil {
return nil, gerror.WrapCode(
gcode.CodeInternalError,
err,
`failed to get primary keys for table`,
)
}
foundPrimaryKey := false
for _, primaryKey := range primaryKeys {
for dataKey := range list[0] {
if strings.EqualFold(dataKey, primaryKey) {
foundPrimaryKey = true
break
}
}
if foundPrimaryKey {
break
}
}
if !foundPrimaryKey {
return nil, gerror.NewCodef(
gcode.CodeMissingParameter,
`Replace/Save/InsertIgnore operation requires conflict detection: `+
`either specify OnConflict() columns or ensure table '%s' has a primary key in the data`,
table,
)
}
// TODO consider composite primary keys.
conflictKeys = primaryKeys
}
var (
one = list[0]
oneLen = len(one)
charL, charR = d.GetChars()
conflictKeySet = gset.New(false)
// queryHolders: Handle data with Holder that need to be merged
// queryValues: Handle data that need to be merged
// insertKeys: Handle valid keys that need to be inserted
// insertValues: Handle values that need to be inserted
// updateValues: Handle values that need to be updated (only when withUpdate=true)
queryHolders = make([]string, oneLen)
queryValues = make([]any, oneLen)
insertKeys = make([]string, oneLen)
insertValues = make([]string, oneLen)
updateValues []string
)
// conflictKeys slice type conv to set type
for _, conflictKey := range conflictKeys {
conflictKeySet.Add(gstr.ToUpper(conflictKey))
}
index := 0
for key, value := range one {
keyWithChar := charL + key + charR
queryHolders[index] = fmt.Sprintf("? AS %s", keyWithChar)
queryValues[index] = value
insertKeys[index] = keyWithChar
insertValues[index] = fmt.Sprintf("T2.%s", keyWithChar)
// Build updateValues only when withUpdate is true
// Filter conflict keys and soft created fields from updateValues
if withUpdate && !(conflictKeySet.Contains(key) || d.Core.IsSoftCreatedFieldName(key)) {
updateValues = append(
updateValues,
fmt.Sprintf(`T1.%s = T2.%s`, keyWithChar, keyWithChar),
)
}
index++
}
var (
batchResult = new(gdb.SqlResult)
sqlStr = parseSqlForMerge(table, queryHolders, insertKeys, insertValues, updateValues, conflictKeys)
)
r, err := d.DoExec(ctx, link, sqlStr, queryValues...)
if err != nil {
return r, err
}
if n, err := r.RowsAffected(); err != nil {
return r, err
} else {
batchResult.Result = r
batchResult.Affected += n
}
return batchResult, nil
}
// parseSqlForMerge generates MERGE statement for DM database.
// When updateValues is empty, it only inserts (INSERT IGNORE behavior).
// When updateValues is provided, it performs upsert (INSERT or UPDATE).
// Examples:
// - INSERT IGNORE: MERGE INTO table T1 USING (...) T2 ON (...) WHEN NOT MATCHED THEN INSERT(...) VALUES (...)
// - UPSERT: MERGE INTO table T1 USING (...) T2 ON (...) WHEN NOT MATCHED THEN INSERT(...) VALUES (...) WHEN MATCHED THEN UPDATE SET ...
func parseSqlForMerge(table string,
queryHolders, insertKeys, insertValues, updateValues, duplicateKey []string,
) (sqlStr string) {
var (
queryHolderStr = strings.Join(queryHolders, ",")
insertKeyStr = strings.Join(insertKeys, ",")
insertValueStr = strings.Join(insertValues, ",")
duplicateKeyStr string
)
// Build ON condition
for index, keys := range duplicateKey {
if index != 0 {
duplicateKeyStr += " AND "
}
duplicateKeyStr += fmt.Sprintf("T1.%s = T2.%s", keys, keys)
}
// Build SQL based on whether UPDATE is needed
pattern := gstr.Trim(`MERGE INTO %s T1 USING (SELECT %s FROM DUAL) T2 ON (%s) WHEN NOT MATCHED THEN INSERT(%s) VALUES (%s)`)
if len(updateValues) > 0 {
// Upsert: INSERT or UPDATE
pattern += gstr.Trim(`WHEN MATCHED THEN UPDATE SET %s`)
return fmt.Sprintf(
pattern, table, queryHolderStr, duplicateKeyStr, insertKeyStr, insertValueStr,
strings.Join(updateValues, ","),
)
}
// Insert Ignore: INSERT only
return fmt.Sprintf(pattern, table, queryHolderStr, duplicateKeyStr, insertKeyStr, insertValueStr)
}