From 4acde0db7b313e7283fe6e351266b5ae6c6e0c90 Mon Sep 17 00:00:00 2001 From: Matthew Herrmann Date: Mon, 21 Feb 2022 09:27:58 +1100 Subject: [PATCH 1/3] Exec() now provides access to status of multiple statements. It now reports the last inserted ID and affected row count for all statements, not just the last one. This is useful to execute batches of statements such as UPDATE with minimal roundtrips. The approach taken is to track last insert id and affected rows using []int64 instead of a int64. Both are set in `mysqlResult`, and a new `mysql.Result` interface makes them accessible to callers calling `Exec()` via `sql.Conn.Raw`. For example: ``` conn.Raw(func(conn interface{}) error { ex := conn.(driver.Execer) res, err := ex.Exec(` UPDATE point SET x = 1 WHERE y = 2; UPDATE point SET x = 2 WHERE y = 3; `, nil) // Both slices have 2 elements. log.Print(res.(mysql.Result).AllRowsAffected()) log.Print(res.(mysql.Result).AllLastInsertIds()) }) ``` --- README.md | 16 +++++++ auth.go | 6 +-- connection.go | 29 ++++++------- driver_test.go | 112 +++++++++++++++++++++++++++++++++++++++++++++++++ infile.go | 8 ++-- packets.go | 77 ++++++++++++++++++++++++++++------ result.go | 37 ++++++++++++++-- rows.go | 7 +++- statement.go | 17 ++++---- 9 files changed, 259 insertions(+), 50 deletions(-) diff --git a/README.md b/README.md index ded6e3b16..e642f33fa 100644 --- a/README.md +++ b/README.md @@ -288,6 +288,22 @@ Allow multiple statements in one query. While this allows batch queries, it also When `multiStatements` is used, `?` parameters must only be used in the first statement. +It's possible to access the last inserted ID and number of affected rows for multiple statements by using `sql.Conn.Raw()` and the `mysql.Result`. For example: + +``` +conn, _ := db.Conn(ctx) +conn.Raw(func(conn interface{}) error { + ex := conn.(driver.Execer) + res, err := ex.Exec(` + UPDATE point SET x = 1 WHERE y = 2; + UPDATE point SET x = 2 WHERE y = 3; + `, nil) + // Both slices have 2 elements. + log.Print(res.(mysql.Result).AllRowsAffected()) + log.Print(res.(mysql.Result).AllLastInsertIds()) +}) +``` + ##### `parseTime` ``` diff --git a/auth.go b/auth.go index a25353429..3c12987ac 100644 --- a/auth.go +++ b/auth.go @@ -347,7 +347,7 @@ func (mc *mysqlConn) handleAuthResult(oldAuthData []byte, plugin string) error { case 1: switch authData[0] { case cachingSha2PasswordFastAuthSuccess: - if err = mc.readResultOK(); err == nil { + if err = mc.resultUnchanged().readResultOK(); err == nil { return nil // auth successful } @@ -398,7 +398,7 @@ func (mc *mysqlConn) handleAuthResult(oldAuthData []byte, plugin string) error { return err } } - return mc.readResultOK() + return mc.resultUnchanged().readResultOK() default: return ErrMalformPkt @@ -427,7 +427,7 @@ func (mc *mysqlConn) handleAuthResult(oldAuthData []byte, plugin string) error { if err != nil { return err } - return mc.readResultOK() + return mc.resultUnchanged().readResultOK() } default: diff --git a/connection.go b/connection.go index 835f89729..9b6126897 100644 --- a/connection.go +++ b/connection.go @@ -23,9 +23,8 @@ import ( type mysqlConn struct { buf buffer netConn net.Conn - rawConn net.Conn // underlying connection when netConn is TLS connection. - affectedRows uint64 - insertId uint64 + rawConn net.Conn // underlying connection when netConn is TLS connection. + result mysqlResult // managed by clearResult() and handleOkPacket(). cfg *Config maxAllowedPacket int maxWriteSize int @@ -149,6 +148,7 @@ func (mc *mysqlConn) cleanup() { if err := mc.netConn.Close(); err != nil { errLog.Print(err) } + mc.clearResult() } func (mc *mysqlConn) error() error { @@ -310,28 +310,25 @@ func (mc *mysqlConn) Exec(query string, args []driver.Value) (driver.Result, err } query = prepared } - mc.affectedRows = 0 - mc.insertId = 0 err := mc.exec(query) if err == nil { - return &mysqlResult{ - affectedRows: int64(mc.affectedRows), - insertId: int64(mc.insertId), - }, err + copied := mc.result + return &copied, err } return nil, mc.markBadConn(err) } // Internal function to execute commands func (mc *mysqlConn) exec(query string) error { + handleOk := mc.clearResult() // Send command if err := mc.writeCommandPacketStr(comQuery, query); err != nil { return mc.markBadConn(err) } // Read Result - resLen, err := mc.readResultSetHeaderPacket() + resLen, err := handleOk.readResultSetHeaderPacket() if err != nil { return err } @@ -348,7 +345,7 @@ func (mc *mysqlConn) exec(query string) error { } } - return mc.discardResults() + return handleOk.discardResults() } func (mc *mysqlConn) Query(query string, args []driver.Value) (driver.Rows, error) { @@ -356,6 +353,8 @@ func (mc *mysqlConn) Query(query string, args []driver.Value) (driver.Rows, erro } func (mc *mysqlConn) query(query string, args []driver.Value) (*textRows, error) { + handleOk := mc.clearResult() + if mc.closed.IsSet() { errLog.Print(ErrInvalidConn) return nil, driver.ErrBadConn @@ -376,7 +375,7 @@ func (mc *mysqlConn) query(query string, args []driver.Value) (*textRows, error) if err == nil { // Read Result var resLen int - resLen, err = mc.readResultSetHeaderPacket() + resLen, err = handleOk.readResultSetHeaderPacket() if err == nil { rows := new(textRows) rows.mc = mc @@ -404,12 +403,13 @@ func (mc *mysqlConn) query(query string, args []driver.Value) (*textRows, error) // The returned byte slice is only valid until the next read func (mc *mysqlConn) getSystemVar(name string) ([]byte, error) { // Send command + handleOk := mc.clearResult() if err := mc.writeCommandPacketStr(comQuery, "SELECT @@"+name); err != nil { return nil, err } // Read Result - resLen, err := mc.readResultSetHeaderPacket() + resLen, err := handleOk.readResultSetHeaderPacket() if err == nil { rows := new(textRows) rows.mc = mc @@ -460,11 +460,12 @@ func (mc *mysqlConn) Ping(ctx context.Context) (err error) { } defer mc.finish() + handleOk := mc.clearResult() if err = mc.writeCommandPacket(comPing); err != nil { return mc.markBadConn(err) } - return mc.readResultOK() + return handleOk.readResultOK() } // BeginTx implements driver.ConnBeginTx interface diff --git a/driver_test.go b/driver_test.go index 4850498d0..8c03e7605 100644 --- a/driver_test.go +++ b/driver_test.go @@ -2155,11 +2155,51 @@ func TestRejectReadOnly(t *testing.T) { } func TestPing(t *testing.T) { + ctx := context.Background() runTests(t, dsn, func(dbt *DBTest) { if err := dbt.db.Ping(); err != nil { dbt.fail("Ping", "Ping", err) } }) + + runTests(t, dsn, func(dbt *DBTest) { + conn, err := dbt.db.Conn(ctx) + if err != nil { + dbt.fail("db", "Conn", err) + } + + // Check that affectedRows and insertIds are cleared after each call. + conn.Raw(func(conn interface{}) error { + c := conn.(*mysqlConn) + + // Issue a query that sets affectedRows and insertIds. + q, err := c.Query(`SELECT 1`, nil) + if err != nil { + dbt.fail("Conn", "Query", err) + } + if got, want := c.result.affectedRows, []int64{0}; !reflect.DeepEqual(got, want) { + dbt.Fatalf("bad affectedRows: got %v, want=%v", got, want) + } + if got, want := c.result.insertIds, []int64{0}; !reflect.DeepEqual(got, want) { + dbt.Fatalf("bad insertIds: got %v, want=%v", got, want) + } + q.Close() + + // Verify that Ping() clears both fields. + for i := 0; i < 2; i++ { + if err := c.Ping(ctx); err != nil { + dbt.fail("Pinger", "Ping", err) + } + if got, want := c.result.affectedRows, []int64(nil); !reflect.DeepEqual(got, want) { + t.Errorf("bad affectedRows: got %v, want=%v", got, want) + } + if got, want := c.result.insertIds, []int64(nil); !reflect.DeepEqual(got, want) { + t.Errorf("bad affectedRows: got %v, want=%v", got, want) + } + } + return nil + }) + }) } // See Issue #799 @@ -2379,6 +2419,42 @@ func TestMultiResultSetNoSelect(t *testing.T) { }) } +func TestExecMultipleResults(t *testing.T) { + ctx := context.Background() + runTestsWithMultiStatement(t, dsn, func(dbt *DBTest) { + dbt.mustExec(` + CREATE TABLE test ( + id INT NOT NULL AUTO_INCREMENT, + value VARCHAR(255), + PRIMARY KEY (id) + )`) + conn, err := dbt.db.Conn(ctx) + if err != nil { + t.Fatalf("failed to connect: %v", err) + } + conn.Raw(func(conn interface{}) error { + ex := conn.(driver.Execer) + res, err := ex.Exec(` + INSERT INTO test (value) VALUES ('a'), ('b'); + INSERT INTO test (value) VALUES ('c'), ('d'), ('e'); + `, nil) + if err != nil { + t.Fatalf("insert statements failed: %v", err) + } + mres := res.(Result) + if got, want := mres.AllRowsAffected(), []int64{2, 3}; !reflect.DeepEqual(got, want) { + t.Errorf("bad AllRowsAffected: got %v, want=%v", got, want) + } + // For INSERTs containing multiple rows, LAST_INSERT_ID() returns the + // first inserted ID, not the last. + if got, want := mres.AllLastInsertIds(), []int64{1, 3}; !reflect.DeepEqual(got, want) { + t.Errorf("bad AllLastInsertIds: got %v, want %v", got, want) + } + return nil + }) + }) +} + // tests if rows are set in a proper state if some results were ignored before // calling rows.NextResultSet. func TestSkipResults(t *testing.T) { @@ -2400,6 +2476,42 @@ func TestSkipResults(t *testing.T) { }) } +func TestQueryMultipleResults(t *testing.T) { + ctx := context.Background() + runTestsWithMultiStatement(t, dsn, func(dbt *DBTest) { + dbt.mustExec(` + CREATE TABLE test ( + id INT NOT NULL AUTO_INCREMENT, + value VARCHAR(255), + PRIMARY KEY (id) + )`) + conn, err := dbt.db.Conn(ctx) + if err != nil { + t.Fatalf("failed to connect: %v", err) + } + conn.Raw(func(conn interface{}) error { + qr := conn.(driver.Queryer) + + c := conn.(*mysqlConn) + + // Demonstrate that repeated queries reset the affectedRows + for i := 0; i < 2; i++ { + _, err := qr.Query(` + INSERT INTO test (value) VALUES ('a'), ('b'); + INSERT INTO test (value) VALUES ('c'), ('d'), ('e'); + `, nil) + if err != nil { + t.Fatalf("insert statements failed: %v", err) + } + if got, want := c.result.affectedRows, []int64{2, 3}; !reflect.DeepEqual(got, want) { + t.Errorf("bad affectedRows: got %v, want=%v", got, want) + } + } + return nil + }) + }) +} + func TestPingContext(t *testing.T) { runTests(t, dsn, func(dbt *DBTest) { ctx, cancel := context.WithCancel(context.Background()) diff --git a/infile.go b/infile.go index 60effdfc2..5b9c3443d 100644 --- a/infile.go +++ b/infile.go @@ -93,7 +93,7 @@ func deferredClose(err *error, closer io.Closer) { } } -func (mc *mysqlConn) handleInFileRequest(name string) (err error) { +func (mc *okHandler) handleInFileRequest(name string) (err error) { var rdr io.Reader var data []byte packetSize := 16 * 1024 // 16KB is small enough for disk readahead and large enough for TCP @@ -154,7 +154,7 @@ func (mc *mysqlConn) handleInFileRequest(name string) (err error) { for err == nil { n, err = rdr.Read(data[4:]) if n > 0 { - if ioErr := mc.writePacket(data[:4+n]); ioErr != nil { + if ioErr := mc.conn().writePacket(data[:4+n]); ioErr != nil { return ioErr } } @@ -168,7 +168,7 @@ func (mc *mysqlConn) handleInFileRequest(name string) (err error) { if data == nil { data = make([]byte, 4) } - if ioErr := mc.writePacket(data[:4]); ioErr != nil { + if ioErr := mc.conn().writePacket(data[:4]); ioErr != nil { return ioErr } @@ -177,6 +177,6 @@ func (mc *mysqlConn) handleInFileRequest(name string) (err error) { return mc.readResultOK() } - mc.readPacket() + mc.conn().readPacket() return err } diff --git a/packets.go b/packets.go index ab30601ae..c73747894 100644 --- a/packets.go +++ b/packets.go @@ -495,7 +495,9 @@ func (mc *mysqlConn) readAuthResult() ([]byte, string, error) { switch data[0] { case iOK: - return nil, "", mc.handleOkPacket(data) + // resultUnchanged, since auth happens before any queries or + // commands have been executed. + return nil, "", mc.resultUnchanged().handleOkPacket(data) case iAuthMoreData: return data[1:], "", err @@ -519,8 +521,8 @@ func (mc *mysqlConn) readAuthResult() ([]byte, string, error) { } // Returns error if Packet is not an 'Result OK'-Packet -func (mc *mysqlConn) readResultOK() error { - data, err := mc.readPacket() +func (mc *okHandler) readResultOK() error { + data, err := mc.conn().readPacket() if err != nil { return err } @@ -528,13 +530,17 @@ func (mc *mysqlConn) readResultOK() error { if data[0] == iOK { return mc.handleOkPacket(data) } - return mc.handleErrorPacket(data) + return mc.conn().handleErrorPacket(data) } // Result Set Header Packet // http://dev.mysql.com/doc/internals/en/com-query-response.html#packet-ProtocolText::Resultset -func (mc *mysqlConn) readResultSetHeaderPacket() (int, error) { - data, err := mc.readPacket() +func (mc *okHandler) readResultSetHeaderPacket() (int, error) { + // handleOkPacket replaces both values; other cases leave the values unchanged. + mc.result.affectedRows = append(mc.result.affectedRows, 0) + mc.result.insertIds = append(mc.result.insertIds, 0) + + data, err := mc.conn().readPacket() if err == nil { switch data[0] { @@ -542,7 +548,7 @@ func (mc *mysqlConn) readResultSetHeaderPacket() (int, error) { return 0, mc.handleOkPacket(data) case iERR: - return 0, mc.handleErrorPacket(data) + return 0, mc.conn().handleErrorPacket(data) case iLocalInFile: return 0, mc.handleInFileRequest(string(data[1:])) @@ -606,18 +612,61 @@ func readStatus(b []byte) statusFlag { return statusFlag(b[0]) | statusFlag(b[1])<<8 } +// Returns an instance of okHandler for codepaths where mysqlConn.result doesn't +// need to be cleared first (eg. during authentication, or while additional +// resultsets are being fetched.) +func (mc *mysqlConn) resultUnchanged() *okHandler { + return (*okHandler)(mc) +} + +// okHandler represents the state of the connection when mysqlConn.result has +// been prepared for processing of OK packets. +// +// To correctly populate mysqlConn.result (updated by handleOkPacket()), all +// callpaths must either: +// +// 1. first clear it using clearResult(), or +// 2. confirm that they don't need to (by calling resultUnchanged()). +// +// Both return an instance of type *okHandler. +type okHandler mysqlConn + +// Exposees the underlying type's methods. +func (mc *okHandler) conn() *mysqlConn { + return (*mysqlConn)(mc) +} + +// clearResult clears the connection's stored affectedRows and insertIds +// fields. +// +// It returns a handler that can process OK responses. +func (mc *mysqlConn) clearResult() *okHandler { + mc.result = mysqlResult{} + return (*okHandler)(mc) +} + // Ok Packet // http://dev.mysql.com/doc/internals/en/generic-response-packets.html#packet-OK_Packet -func (mc *mysqlConn) handleOkPacket(data []byte) error { +func (mc *okHandler) handleOkPacket(data []byte) error { var n, m int + var affectedRows, insertId uint64 // 0x00 [1 byte] // Affected rows [Length Coded Binary] - mc.affectedRows, _, n = readLengthEncodedInteger(data[1:]) + affectedRows, _, n = readLengthEncodedInteger(data[1:]) // Insert id [Length Coded Binary] - mc.insertId, _, m = readLengthEncodedInteger(data[1+n:]) + insertId, _, m = readLengthEncodedInteger(data[1+n:]) + + // Update for the current statement result (only used by + // readResultSetHeaderPacket). + if len(mc.result.affectedRows) > 0 { + mc.result.affectedRows[len(mc.result.affectedRows)-1] = int64(affectedRows) + } + if len(mc.result.insertIds) > 0 { + mc.result.insertIds[len(mc.result.insertIds)-1] = int64(insertId) + } // server_status [2 bytes] mc.status = readStatus(data[1+n+m : 1+n+m+2]) @@ -1148,7 +1197,9 @@ func (stmt *mysqlStmt) writeExecutePacket(args []driver.Value) error { return mc.writePacket(data) } -func (mc *mysqlConn) discardResults() error { +// For each remaining resultset in the stream, discards its rows and updates +// mc.affectedRows and mc.insertIds. +func (mc *okHandler) discardResults() error { for mc.status&statusMoreResultsExists != 0 { resLen, err := mc.readResultSetHeaderPacket() if err != nil { @@ -1156,11 +1207,11 @@ func (mc *mysqlConn) discardResults() error { } if resLen > 0 { // columns - if err := mc.readUntilEOF(); err != nil { + if err := mc.conn().readUntilEOF(); err != nil { return err } // rows - if err := mc.readUntilEOF(); err != nil { + if err := mc.conn().readUntilEOF(); err != nil { return err } } diff --git a/result.go b/result.go index c6438d034..36a432e81 100644 --- a/result.go +++ b/result.go @@ -8,15 +8,44 @@ package mysql +import "database/sql/driver" + +// Result exposes data not available through *connection.Result. +// +// This is accessible by executing statements using sql.Conn.Raw() and +// downcasting the returned result: +// +// res, err := rawConn.Exec(...) +// res.(mysql.Result).AllRowsAffected() +// +type Result interface { + driver.Result + // AllRowsAffected returns a slice containing the affected rows for each + // executed statement. + AllRowsAffected() []int64 + // AllLastInsertIds returns a slice containing the last inserted ID for each + // executed statement. + AllLastInsertIds() []int64 +} + type mysqlResult struct { - affectedRows int64 - insertId int64 + // One entry in both slices is created for every executed statement result. + affectedRows []int64 + insertIds []int64 } func (res *mysqlResult) LastInsertId() (int64, error) { - return res.insertId, nil + return res.insertIds[len(res.insertIds)-1], nil } func (res *mysqlResult) RowsAffected() (int64, error) { - return res.affectedRows, nil + return res.affectedRows[len(res.affectedRows)-1], nil +} + +func (res *mysqlResult) AllLastInsertIds() []int64 { + return append([]int64{}, res.insertIds...) // defensive copy +} + +func (res *mysqlResult) AllRowsAffected() []int64 { + return append([]int64{}, res.affectedRows...) // defensive copy } diff --git a/rows.go b/rows.go index 888bdb5f0..63d0ed2d5 100644 --- a/rows.go +++ b/rows.go @@ -123,7 +123,8 @@ func (rows *mysqlRows) Close() (err error) { err = mc.readUntilEOF() } if err == nil { - if err = mc.discardResults(); err != nil { + handleOk := mc.clearResult() + if err = handleOk.discardResults(); err != nil { return err } } @@ -160,7 +161,9 @@ func (rows *mysqlRows) nextResultSet() (int, error) { return 0, io.EOF } rows.rs = resultSet{} - return rows.mc.readResultSetHeaderPacket() + // rows.mc.affectedRows and rows.mc.insertIds accumulate on each call to + // nextResultSet. + return rows.mc.resultUnchanged().readResultSetHeaderPacket() } func (rows *mysqlRows) nextNotEmptyResultSet() (int, error) { diff --git a/statement.go b/statement.go index 18a3ae498..5b053f38a 100644 --- a/statement.go +++ b/statement.go @@ -61,12 +61,10 @@ func (stmt *mysqlStmt) Exec(args []driver.Value) (driver.Result, error) { } mc := stmt.mc - - mc.affectedRows = 0 - mc.insertId = 0 + handleOk := stmt.mc.clearResult() // Read Result - resLen, err := mc.readResultSetHeaderPacket() + resLen, err := handleOk.readResultSetHeaderPacket() if err != nil { return nil, err } @@ -83,14 +81,12 @@ func (stmt *mysqlStmt) Exec(args []driver.Value) (driver.Result, error) { } } - if err := mc.discardResults(); err != nil { + if err := handleOk.discardResults(); err != nil { return nil, err } - return &mysqlResult{ - affectedRows: int64(mc.affectedRows), - insertId: int64(mc.insertId), - }, nil + copied := mc.result + return &copied, nil } func (stmt *mysqlStmt) Query(args []driver.Value) (driver.Rows, error) { @@ -111,7 +107,8 @@ func (stmt *mysqlStmt) query(args []driver.Value) (*binaryRows, error) { mc := stmt.mc // Read Result - resLen, err := mc.readResultSetHeaderPacket() + handleOk := stmt.mc.clearResult() + resLen, err := handleOk.readResultSetHeaderPacket() if err != nil { return nil, err } From 621c15078306c3bb1c36034815a148eb5ed54942 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 29 May 2023 04:04:50 +0900 Subject: [PATCH 2/3] Apply suggestions from code review --- packets.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packets.go b/packets.go index dc23ff6b9..1a7f2c376 100644 --- a/packets.go +++ b/packets.go @@ -630,7 +630,7 @@ func readStatus(b []byte) statusFlag { } // Returns an instance of okHandler for codepaths where mysqlConn.result doesn't -// need to be cleared first (eg. during authentication, or while additional +// need to be cleared first (e.g. during authentication, or while additional // resultsets are being fetched.) func (mc *mysqlConn) resultUnchanged() *okHandler { return (*okHandler)(mc) From 3eb80e098b7face1769e5cf9afb8f573020683e0 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 29 May 2023 12:19:21 +0900 Subject: [PATCH 3/3] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2b02101c9..4eade6853 100644 --- a/README.md +++ b/README.md @@ -307,7 +307,7 @@ When `multiStatements` is used, `?` parameters must only be used in the first st It's possible to access the last inserted ID and number of affected rows for multiple statements by using `sql.Conn.Raw()` and the `mysql.Result`. For example: -``` +```go conn, _ := db.Conn(ctx) conn.Raw(func(conn interface{}) error { ex := conn.(driver.Execer)