From 99536c8bef344299360351277210eeebb84fd021 Mon Sep 17 00:00:00 2001 From: John Guo Date: Mon, 8 Dec 2025 14:37:35 +0800 Subject: [PATCH] up --- .github/workflows/ci-main.yml | 12 ++--- .../clickhouse/clickhouse_do_insert.go | 1 + contrib/drivers/dm/dm_do_insert.go | 38 +++++++-------- contrib/drivers/mssql/mssql_do_exec.go | 4 +- contrib/drivers/mssql/mssql_do_insert.go | 7 +-- contrib/drivers/oracle/oracle_do_insert.go | 17 +++---- contrib/drivers/pgsql/pgsql_do_insert.go | 23 +++++++-- contrib/drivers/pgsql/pgsql_table_fields.go | 14 +++++- contrib/drivers/pgsql/pgsql_z_unit_db_test.go | 14 +++--- .../drivers/pgsql/pgsql_z_unit_field_test.go | 16 +++---- .../drivers/pgsql/pgsql_z_unit_model_test.go | 47 +++++++++++++++++++ .../drivers/pgsql/pgsql_z_unit_upsert_test.go | 2 +- database/gdb/gdb_driver_wrapper_db.go | 12 ++++- 13 files changed, 137 insertions(+), 70 deletions(-) diff --git a/.github/workflows/ci-main.yml b/.github/workflows/ci-main.yml index e3493de60..479f60176 100644 --- a/.github/workflows/ci-main.yml +++ b/.github/workflows/ci-main.yml @@ -54,7 +54,7 @@ jobs: # Service containers to run with `code-test` services: # Etcd service. - # docker run -d --name etcd -p 2379:2379 -e ALLOW_NONE_AUTHENTICATION=yes bitnamilegacy/etcd:3.4.24 + # docker run --name etcd -p 2379:2379 -e ALLOW_NONE_AUTHENTICATION=yes bitnamilegacy/etcd:3.4.24 etcd: image: bitnamilegacy/etcd:3.4.24 env: @@ -75,7 +75,7 @@ jobs: - 6379:6379 # MySQL backend server. - # docker run -d --name mysql \ + # docker run --name mysql \ # -p 3306:3306 \ # -e MYSQL_DATABASE=test \ # -e MYSQL_ROOT_PASSWORD=12345678 \ @@ -89,7 +89,7 @@ jobs: - 3306:3306 # MariaDb backend server. - # docker run -d --name mariadb \ + # docker run --name mariadb \ # -p 3307:3306 \ # -e MYSQL_DATABASE=test \ # -e MYSQL_ROOT_PASSWORD=12345678 \ @@ -103,7 +103,7 @@ jobs: - 3307:3306 # PostgreSQL backend server. - # docker run -d --name postgres \ + # docker run --name postgres \ # -p 5432:5432 \ # -e POSTGRES_PASSWORD=12345678 \ # -e POSTGRES_USER=postgres \ @@ -150,7 +150,7 @@ jobs: --health-retries 10 # ClickHouse backend server. - # docker run -d --name clickhouse \ + # docker run --name clickhouse \ # -p 9000:9000 -p 8123:8123 -p 9001:9001 \ # clickhouse/clickhouse-server:24.11.1.2557-alpine clickhouse-server: @@ -161,7 +161,7 @@ jobs: - 9001:9001 # Polaris backend server. - # docker run -d --name polaris \ + # docker run --name polaris \ # -p 8090:8090 -p 8091:8091 -p 8093:8093 -p 9090:9090 -p 9091:9091 \ # polarismesh/polaris-standalone:v1.17.2 polaris: diff --git a/contrib/drivers/clickhouse/clickhouse_do_insert.go b/contrib/drivers/clickhouse/clickhouse_do_insert.go index a6c397ae3..a71276913 100644 --- a/contrib/drivers/clickhouse/clickhouse_do_insert.go +++ b/contrib/drivers/clickhouse/clickhouse_do_insert.go @@ -16,6 +16,7 @@ import ( ) // DoInsert inserts or updates data for given table. +// The list parameter must contain at least one record, which was previously validated. func (d *Driver) DoInsert( ctx context.Context, link gdb.Link, table string, list gdb.List, option gdb.DoInsertOption, ) (result sql.Result, err error) { diff --git a/contrib/drivers/dm/dm_do_insert.go b/contrib/drivers/dm/dm_do_insert.go index 022c6fcef..b5ccb336c 100644 --- a/contrib/drivers/dm/dm_do_insert.go +++ b/contrib/drivers/dm/dm_do_insert.go @@ -20,6 +20,7 @@ import ( ) // DoInsert inserts or updates data for given table. +// The list parameter must contain at least one record, which was previously validated. func (d *Driver) DoInsert( ctx context.Context, link gdb.Link, table string, list gdb.List, option gdb.DoInsertOption, ) (result sql.Result, err error) { @@ -60,16 +61,12 @@ func (d *Driver) doInsertIgnore(ctx context.Context, // When withUpdate is false, it performs insert ignore (insert only when no conflict). func (d *Driver) doMergeInsert( ctx context.Context, - link gdb.Link, - table string, - list gdb.List, - option gdb.DoInsertOption, - withUpdate bool, + link gdb.Link, table string, list gdb.List, option gdb.DoInsertOption, withUpdate bool, ) (result sql.Result, err error) { // If OnConflict is not specified, automatically get the primary key of the table conflictKeys := option.OnConflict if len(conflictKeys) == 0 { - conflictKeys, err = d.getPrimaryKeys(ctx, table) + primaryKeys, err := d.getPrimaryKeys(ctx, table) if err != nil { return nil, gerror.WrapCode( gcode.CodeInternalError, @@ -77,29 +74,26 @@ func (d *Driver) doMergeInsert( `failed to get primary keys for table`, ) } - if len(conflictKeys) == 0 { + foundPrimaryKey := false + for _, primaryKey := range primaryKeys { + if _, ok := list[0][primaryKey]; ok { + foundPrimaryKey = true + break + } + } + if !foundPrimaryKey { return nil, gerror.NewCode( gcode.CodeMissingParameter, - `Please specify conflict columns or ensure the table has a primary key`, + `Please specify conflict columns or ensure the record has a primary key for Save/Replace/InsertIgnore operation`, ) } - } - - if len(list) == 0 { - opName := "Save" - if !withUpdate { - opName = "InsertIgnore" - } - return nil, gerror.NewCodef( - gcode.CodeInvalidRequest, `%s operation list is empty by dm driver`, opName, - ) + conflictKeys = primaryKeys } var ( - one = list[0] - oneLen = len(one) - charL, charR = d.GetChars() - + one = list[0] + oneLen = len(one) + charL, charR = d.GetChars() conflictKeySet = gset.New(false) // queryHolders: Handle data with Holder that need to be merged diff --git a/contrib/drivers/mssql/mssql_do_exec.go b/contrib/drivers/mssql/mssql_do_exec.go index b8b014244..275a8bb15 100644 --- a/contrib/drivers/mssql/mssql_do_exec.go +++ b/contrib/drivers/mssql/mssql_do_exec.go @@ -167,8 +167,8 @@ func (r *InsertResult) RowsAffected() (int64, error) { } // GetInsertOutputSql gen get last_insert_id code -func (m *Driver) GetInsertOutputSql(ctx context.Context, table string) string { - fds, errFd := m.GetDB().TableFields(ctx, table) +func (d *Driver) GetInsertOutputSql(ctx context.Context, table string) string { + fds, errFd := d.GetDB().TableFields(ctx, table) if errFd != nil { return "" } diff --git a/contrib/drivers/mssql/mssql_do_insert.go b/contrib/drivers/mssql/mssql_do_insert.go index 5e467d730..4284780b7 100644 --- a/contrib/drivers/mssql/mssql_do_insert.go +++ b/contrib/drivers/mssql/mssql_do_insert.go @@ -20,6 +20,7 @@ import ( ) // DoInsert inserts or updates data for given table. +// The list parameter must contain at least one record, which was previously validated. func (d *Driver) DoInsert(ctx context.Context, link gdb.Link, table string, list gdb.List, option gdb.DoInsertOption) (result sql.Result, err error) { switch option.InsertOption { case gdb.InsertOptionSave: @@ -46,12 +47,6 @@ func (d *Driver) doSave(ctx context.Context, ) } - if len(list) == 0 { - return nil, gerror.NewCode( - gcode.CodeInvalidRequest, `Save operation list is empty by mssql driver`, - ) - } - var ( one = list[0] oneLen = len(one) diff --git a/contrib/drivers/oracle/oracle_do_insert.go b/contrib/drivers/oracle/oracle_do_insert.go index d59bdf95b..4bf89a80f 100644 --- a/contrib/drivers/oracle/oracle_do_insert.go +++ b/contrib/drivers/oracle/oracle_do_insert.go @@ -21,6 +21,7 @@ import ( ) // DoInsert inserts or updates data for given table. +// The list parameter must contain at least one record, which was previously validated. func (d *Driver) DoInsert( ctx context.Context, link gdb.Link, table string, list gdb.List, option gdb.DoInsertOption, ) (result sql.Result, err error) { @@ -33,6 +34,7 @@ func (d *Driver) DoInsert( gcode.CodeNotSupported, `Replace operation is not supported by oracle driver`, ) + default: } var ( keys []string @@ -93,7 +95,7 @@ func (d *Driver) DoInsert( return batchResult, nil } -// doSave support upsert for Oracle +// doSave support upsert for Oracle. func (d *Driver) doSave(ctx context.Context, link gdb.Link, table string, list gdb.List, option gdb.DoInsertOption, ) (result sql.Result, err error) { @@ -103,17 +105,10 @@ func (d *Driver) doSave(ctx context.Context, ) } - if len(list) == 0 { - return nil, gerror.NewCode( - gcode.CodeInvalidRequest, `Save operation list is empty by oracle driver`, - ) - } - var ( - one = list[0] - oneLen = len(one) - charL, charR = d.GetChars() - + one = list[0] + oneLen = len(one) + charL, charR = d.GetChars() conflictKeys = option.OnConflict conflictKeySet = gset.New(false) diff --git a/contrib/drivers/pgsql/pgsql_do_insert.go b/contrib/drivers/pgsql/pgsql_do_insert.go index e5ca80f2a..570ac27c3 100644 --- a/contrib/drivers/pgsql/pgsql_do_insert.go +++ b/contrib/drivers/pgsql/pgsql_do_insert.go @@ -17,9 +17,15 @@ import ( ) // DoInsert inserts or updates data for given table. -func (d *Driver) DoInsert(ctx context.Context, link gdb.Link, table string, list gdb.List, option gdb.DoInsertOption) (result sql.Result, err error) { +// The list parameter must contain at least one record, which was previously validated. +func (d *Driver) DoInsert( + ctx context.Context, + link gdb.Link, table string, list gdb.List, option gdb.DoInsertOption, +) (result sql.Result, err error) { switch option.InsertOption { - case gdb.InsertOptionReplace: + case + gdb.InsertOptionReplace, + gdb.InsertOptionSave: // PostgreSQL does not support REPLACE INTO syntax, use Save (ON CONFLICT ... DO UPDATE) instead. // Automatically detect primary keys if OnConflict is not specified. if len(option.OnConflict) == 0 { @@ -31,10 +37,17 @@ func (d *Driver) DoInsert(ctx context.Context, link gdb.Link, table string, list `failed to get primary keys for Replace operation`, ) } - if len(primaryKeys) == 0 { + foundPrimaryKey := false + for _, conflictKey := range primaryKeys { + if _, ok := list[0][conflictKey]; ok { + foundPrimaryKey = true + break + } + } + if !foundPrimaryKey { return nil, gerror.NewCode( gcode.CodeMissingParameter, - `Replace operation requires primary key, but table has no primary key defined`, + `Please specify conflict columns or ensure the record has a primary key for Save/Replace operation`, ) } option.OnConflict = primaryKeys @@ -46,7 +59,7 @@ func (d *Driver) DoInsert(ctx context.Context, link gdb.Link, table string, list tableFields, err := d.GetCore().GetDB().TableFields(ctx, table) if err == nil { for _, field := range tableFields { - if field.Key == "pri" { + if gstr.Equal(field.Key, "PRI") { pkField := *field ctx = context.WithValue(ctx, internalPrimaryKeyInCtx, pkField) break diff --git a/contrib/drivers/pgsql/pgsql_table_fields.go b/contrib/drivers/pgsql/pgsql_table_fields.go index 07f3a4e43..8573648f6 100644 --- a/contrib/drivers/pgsql/pgsql_table_fields.go +++ b/contrib/drivers/pgsql/pgsql_table_fields.go @@ -80,10 +80,22 @@ func (d *Driver) TableFields(ctx context.Context, table string, schema ...string } continue } + + var ( + fieldType string + dataType = m["type"].String() + dataLength = m["length"].Int() + ) + if dataLength > 0 { + fieldType = fmt.Sprintf("%s(%d)", dataType, dataLength) + } else { + fieldType = dataType + } + fields[name] = &gdb.TableField{ Index: index, Name: name, - Type: m["type"].String(), + Type: fieldType, Null: !m["null"].Bool(), Key: m["key"].String(), Default: m["default_value"].Val(), diff --git a/contrib/drivers/pgsql/pgsql_z_unit_db_test.go b/contrib/drivers/pgsql/pgsql_z_unit_db_test.go index 12366f629..a83cb38bf 100644 --- a/contrib/drivers/pgsql/pgsql_z_unit_db_test.go +++ b/contrib/drivers/pgsql/pgsql_z_unit_db_test.go @@ -90,7 +90,7 @@ func Test_DB_Save(t *testing.T) { "create_time": gtime.Now().String(), } _, err := db.Save(ctx, "t_user", data, 10) - gtest.AssertNE(err, nil) + gtest.AssertNil(err) }) } @@ -323,10 +323,10 @@ func Test_DB_TableFields(t *testing.T) { var expect = map[string][]any{ // []string: Index Type Null Key Default Comment // id is bigserial so the default is a pgsql function - "id": {0, "int8", false, "pri", fmt.Sprintf("nextval('%s_id_seq'::regclass)", table), ""}, - "passport": {1, "varchar", false, "", nil, ""}, - "password": {2, "varchar", false, "", nil, ""}, - "nickname": {3, "varchar", false, "", nil, ""}, + "id": {0, "int8(64)", false, "pri", fmt.Sprintf("nextval('%s_id_seq'::regclass)", table), ""}, + "passport": {1, "varchar(45)", false, "", nil, ""}, + "password": {2, "varchar(32)", false, "", nil, ""}, + "nickname": {3, "varchar(45)", false, "", nil, ""}, "create_time": {4, "timestamp", false, "", nil, ""}, } @@ -429,13 +429,13 @@ func Test_DB_TableFields_DuplicateConstraints(t *testing.T) { t.AssertNE(fields["id"], nil) t.Assert(fields["id"].Key, "pri") t.Assert(fields["id"].Name, "id") - t.Assert(fields["id"].Type, "int8") + t.Assert(fields["id"].Type, "int8(64)") // Verify email field has unique constraint t.AssertNE(fields["email"], nil) t.Assert(fields["email"].Key, "uni") t.Assert(fields["email"].Name, "email") - t.Assert(fields["email"].Type, "varchar") + t.Assert(fields["email"].Type, "varchar(100)") // Verify username field has no constraint t.AssertNE(fields["username"], nil) diff --git a/contrib/drivers/pgsql/pgsql_z_unit_field_test.go b/contrib/drivers/pgsql/pgsql_z_unit_field_test.go index 7c3df4ab0..22a8f25a2 100644 --- a/contrib/drivers/pgsql/pgsql_z_unit_field_test.go +++ b/contrib/drivers/pgsql/pgsql_z_unit_field_test.go @@ -73,18 +73,18 @@ func Test_TableFields_Types(t *testing.T) { t.AssertNil(err) // Test integer type names - t.Assert(fields["col_int2"].Type, "int2") - t.Assert(fields["col_int4"].Type, "int4") - t.Assert(fields["col_int8"].Type, "int8") + t.Assert(fields["col_int2"].Type, "int2(16)") + t.Assert(fields["col_int4"].Type, "int4(32)") + t.Assert(fields["col_int8"].Type, "int8(64)") // Test float type names - t.Assert(fields["col_float4"].Type, "float4") - t.Assert(fields["col_float8"].Type, "float8") - t.Assert(fields["col_numeric"].Type, "numeric") + t.Assert(fields["col_float4"].Type, "float4(24)") + t.Assert(fields["col_float8"].Type, "float8(53)") + t.Assert(fields["col_numeric"].Type, "numeric(10)") // Test character type names - t.Assert(fields["col_char"].Type, "bpchar") - t.Assert(fields["col_varchar"].Type, "varchar") + t.Assert(fields["col_char"].Type, "bpchar(10)") + t.Assert(fields["col_varchar"].Type, "varchar(100)") t.Assert(fields["col_text"].Type, "text") // Test boolean type name diff --git a/contrib/drivers/pgsql/pgsql_z_unit_model_test.go b/contrib/drivers/pgsql/pgsql_z_unit_model_test.go index 2134a91bf..f971ff2f2 100644 --- a/contrib/drivers/pgsql/pgsql_z_unit_model_test.go +++ b/contrib/drivers/pgsql/pgsql_z_unit_model_test.go @@ -796,3 +796,50 @@ func Test_ConvertSliceFloat64(t *testing.T) { }) } } + +func Test_Model_InsertIgnore(t *testing.T) { + table := createTable() + defer dropTable(table) + + gtest.C(t, func(t *gtest.T) { + user := db.Model(table) + result, err := user.Data(g.Map{ + "id": 1, + "uid": 1, + "passport": "t1", + "password": "25d55ad283aa400af464c76d713c07ad", + "nickname": "name_1", + "create_time": gtime.Now().String(), + }).Insert() + t.AssertNil(err) + n, _ := result.RowsAffected() + t.Assert(n, 1) + + result, err = db.Model(table).Data(g.Map{ + "id": 1, + "uid": 1, + "passport": "t1", + "password": "25d55ad283aa400af464c76d713c07ad", + "nickname": "name_1", + "create_time": gtime.Now().String(), + }).Insert() + t.AssertNE(err, nil) + + result, err = db.Model(table).Data(g.Map{ + "id": 1, + "uid": 1, + "passport": "t2", + "password": "25d55ad283aa400af464c76d713c07ad", + "nickname": "name_2", + "create_time": gtime.Now().String(), + }).InsertIgnore() + t.AssertNil(err) + + n, _ = result.RowsAffected() + t.Assert(n, 0) + + value, err := db.Model(table).Fields("passport").WherePri(1).Value() + t.AssertNil(err) + t.Assert(value.String(), "t1") + }) +} diff --git a/contrib/drivers/pgsql/pgsql_z_unit_upsert_test.go b/contrib/drivers/pgsql/pgsql_z_unit_upsert_test.go index a93017e97..aff3f7081 100644 --- a/contrib/drivers/pgsql/pgsql_z_unit_upsert_test.go +++ b/contrib/drivers/pgsql/pgsql_z_unit_upsert_test.go @@ -222,7 +222,7 @@ func Test_FormatUpsert_NoOnConflict(t *testing.T) { // Try Save without OnConflict - should fail for pgsql // PostgreSQL requires OnConflict() for Save() operations, unlike MySQL _, err = db.Model(table).Data(g.Map{ - "id": 1, + // "id": 1, "passport": "no_conflict_user", "password": "newpwd", "nickname": "newnick", diff --git a/database/gdb/gdb_driver_wrapper_db.go b/database/gdb/gdb_driver_wrapper_db.go index 7dbc1d0ce..81c5b729c 100644 --- a/database/gdb/gdb_driver_wrapper_db.go +++ b/database/gdb/gdb_driver_wrapper_db.go @@ -109,7 +109,17 @@ func (d *DriverWrapperDB) TableFields( // InsertOptionReplace: if there's unique/primary key in the data, it deletes it from table and inserts a new one; // InsertOptionSave: if there's unique/primary key in the data, it updates it or else inserts a new one; // InsertOptionIgnore: if there's unique/primary key in the data, it ignores the inserting; -func (d *DriverWrapperDB) DoInsert(ctx context.Context, link Link, table string, list List, option DoInsertOption) (result sql.Result, err error) { +func (d *DriverWrapperDB) DoInsert( + ctx context.Context, link Link, table string, list List, option DoInsertOption, +) (result sql.Result, err error) { + if len(list) == 0 { + return nil, gerror.NewCodef( + gcode.CodeInvalidRequest, + `data list is empty for %s operation`, + GetInsertOperationByOption(option.InsertOption), + ) + } + // Convert data type before commit it to underlying db driver. for i, item := range list { list[i], err = d.GetCore().ConvertDataForRecord(ctx, item, table)