Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Handle MySQL handler error codes #17252

Merged
merged 4 commits into from
Nov 19, 2024
Merged

Handle MySQL handler error codes #17252

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
c3ae17f
Handle MySQL handler error codes
shlomi-noach Nov 18, 2024
2ccef16
full HA Error listing
shlomi-noach Nov 19, 2024
cea91ee
Opt-in for recoverable errors rather than opt-out
shlomi-noach Nov 19, 2024
cb99a24
exlude zero error
shlomi-noach Nov 19, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 21 additions & 0 deletions go/mysql/sqlerror/constants.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -203,6 +203,7 @@ const (
ERBlobKeyWithoutLength = ErrorCode(1170)
ERPrimaryCantHaveNull = ErrorCode(1171)
ERTooManyRows = ErrorCode(1172)
ERErrorDuringCommit = ErrorCode(1180)
ERLockOrActiveTransaction = ErrorCode(1192)
ERUnknownSystemVariable = ErrorCode(1193)
ERSetConstantsOnly = ErrorCode(1204)
Expand Down Expand Up @@ -301,6 +302,26 @@ const (
ERServerIsntAvailable = ErrorCode(3168)
)

// HandlerErrorCode is for errors thrown by the handler, and which are then embedded in other errors.
type HandlerErrorCode uint16

func (e HandlerErrorCode) ToString() string {
return strconv.FormatUint(uint64(e), 10)
}

const (
HaErrCrashed = HandlerErrorCode(126)
HaErrTooBigRow = HandlerErrorCode(139)
HaErrFoundDuppUnique = HandlerErrorCode(141)
HaErrLockWaitTimeout = HandlerErrorCode(146)
HaErrLockTableFull = HandlerErrorCode(147)
HaErrLockDeadlock = HandlerErrorCode(149)
HaErrTooManyConcurrentTrxs = HandlerErrorCode(177)
HaErrNotInLockPartitions = HandlerErrorCode(178)
HaErrQueryInterrupted = HandlerErrorCode(196)
HaErrDiskFullNowait = HandlerErrorCode(204)
)

// Sql states for errors.
// Originally found in include/mysql/sql_state.h
const (
Expand Down
12 changes: 12 additions & 0 deletions go/mysql/sqlerror/sql_error.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,18 @@ func NewSQLError(number ErrorCode, sqlState string, msg string) *SQLError {
}
}

var handlerErrExtract = regexp.MustCompile(`Got error ([0-9]*) [-] .* (from storage engine|during COMMIT|during ROLLBACK)`)

func (se *SQLError) HaErrorCode() HandlerErrorCode {
match := handlerErrExtract.FindStringSubmatch(se.Message)
if len(match) >= 1 {
if code, err := strconv.ParseUint(match[1], 10, 16); err == nil {
return HandlerErrorCode(code)
}
}
return 0
}

// Error implements the error interface
func (se *SQLError) Error() string {
var buf strings.Builder
Expand Down
21 changes: 21 additions & 0 deletions go/mysql/sqlerror/sql_error_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ func TestNewSQLErrorFromError(t *testing.T) {
var tCases = []struct {
err error
num ErrorCode
ha HandlerErrorCode
ss string
}{
{
Expand Down Expand Up @@ -179,6 +180,24 @@ func TestNewSQLErrorFromError(t *testing.T) {
num: ERDupEntry,
ss: SSConstraintViolation,
},
{
err: fmt.Errorf("ERROR HY000: Got error 204 - 'No more room in disk' during COMMIT"),
num: ERUnknownError,
ss: SSUnknownSQLState,
ha: HaErrDiskFullNowait,
},
{
err: fmt.Errorf("COMMIT failed w/ error: Got error 204 - 'No more room in disk' during COMMIT (errno 1180) (sqlstate HY000) during query: commit"),
num: ERErrorDuringCommit,
ss: SSUnknownSQLState,
ha: HaErrDiskFullNowait,
},
{
err: fmt.Errorf("COMMIT failed w/ error: Got error 149 - 'Lock deadlock; Retry transaction' during COMMIT (errno 1180) (sqlstate HY000) during query: commit"),
num: ERErrorDuringCommit,
ss: SSUnknownSQLState,
ha: HaErrLockDeadlock,
},
}

for _, tc := range tCases {
Expand All @@ -187,6 +206,8 @@ func TestNewSQLErrorFromError(t *testing.T) {
require.ErrorAs(t, NewSQLErrorFromError(tc.err), &err)
assert.Equal(t, tc.num, err.Number())
assert.Equal(t, tc.ss, err.SQLState())
ha := err.HaErrorCode()
assert.Equal(t, tc.ha, ha)
})
}
}
10 changes: 10 additions & 0 deletions go/vt/vttablet/tabletmanager/vreplication/utils.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -232,6 +232,16 @@ func isUnrecoverableError(err error) bool {
sqlerror.ERWrongValueCountOnRow:
log.Errorf("Got unrecoverable error: %v", sqlErr)
return true
case sqlerror.ERErrorDuringCommit:
switch sqlErr.HaErrorCode() {
case
sqlerror.HaErrCrashed,
sqlerror.HaErrTooBigRow,
sqlerror.HaErrFoundDuppUnique,
sqlerror.HaErrDiskFullNowait:
log.Errorf("Got unrecoverable error: %v", sqlErr)
return true
}
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@shlomi-noach Isn't it easier to invert this and only here have explicit error codes that are recoverable? That's I think also more robust for new error codes etc. in the future to assume it's not recoverable?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm ambivalent. On one hand, until today I have never encountered this error in Vitess. On the other hand, I'm afraid to introduce new (so called "unaccounted for") failure points.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'd lean towards treating unknown things then as not recoverable / retryable? Seems less risky to keep hammering on something that is an unexpected / unknown failure?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We now opt-in for recoverable errors.

}
return false
}
10 changes: 10 additions & 0 deletions go/vt/vttablet/tabletmanager/vreplication/utils_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -152,6 +152,16 @@ func TestIsUnrecoverableError(t *testing.T) {
err: sqlerror.NewSQLError(sqlerror.ERDataOutOfRange, "data out of range", "test"),
expected: true,
},
{
name: "SQL error with HaErrDiskFullNowait error",
err: sqlerror.NewSQLError(sqlerror.ERErrorDuringCommit, "unknown", "ERROR HY000: Got error 204 - 'No more room in disk' during COMMIT"),
expected: true,
},
{
name: "SQL error with HaErrLockDeadlock error",
err: sqlerror.NewSQLError(sqlerror.ERErrorDuringCommit, "unknown", "ERROR HY000: Got error 149 - 'Lock deadlock; Retry transaction' during COMMIT"),
expected: false,
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
Expand Down
Loading