Skip to content

Commit d5e78e4

Browse files
author
Shlomi Noach
authored
Merge pull request #277 from github/enum-sort-test
WIP testing enum as part of PK
2 parents b718bf6 + ac61597 commit d5e78e4

7 files changed

Lines changed: 87 additions & 41 deletions

File tree

‎doc/requirements-and-limitations.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,3 +43,4 @@ The `SUPER` privilege is required for `STOP SLAVE`, `START SLAVE` operations. Th
4343
- Multisource is not supported when migrating via replica. It _should_ work (but never tested) when connecting directly to master (`--allow-on-master`)
4444

4545
- Master-master setup is only supported in active-passive setup. Active-active (where table is being written to on both masters concurrently) is unsupported. It may be supported in the future.
46+
- If you have en `enum` field as part of your migration key (typically the `PRIMARY KEY`), migration performance will be degraded and potentially bad. [Read more](https://github.com/github/gh-ost/pull/277#issuecomment-254811520)

‎go/logic/applier.go‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -328,7 +328,7 @@ func (this *Applier) ExecuteThrottleQuery() (int64, error) {
328328
// ReadMigrationMinValues returns the minimum values to be iterated on rowcopy
329329
func (this *Applier) ReadMigrationMinValues(uniqueKey *sql.UniqueKey) error {
330330
log.Debugf("Reading migration range according to key: %s", uniqueKey.Name)
331-
query, err := sql.BuildUniqueKeyMinValuesPreparedQuery(this.migrationContext.DatabaseName, this.migrationContext.OriginalTableName, uniqueKey.Columns.Names())
331+
query, err := sql.BuildUniqueKeyMinValuesPreparedQuery(this.migrationContext.DatabaseName, this.migrationContext.OriginalTableName, &uniqueKey.Columns)
332332
if err != nil {
333333
return err
334334
}
@@ -349,7 +349,7 @@ func (this *Applier) ReadMigrationMinValues(uniqueKey *sql.UniqueKey) error {
349349
// ReadMigrationMaxValues returns the maximum values to be iterated on rowcopy
350350
func (this *Applier) ReadMigrationMaxValues(uniqueKey *sql.UniqueKey) error {
351351
log.Debugf("Reading migration range according to key: %s", uniqueKey.Name)
352-
query, err := sql.BuildUniqueKeyMaxValuesPreparedQuery(this.migrationContext.DatabaseName, this.migrationContext.OriginalTableName, uniqueKey.Columns.Names())
352+
query, err := sql.BuildUniqueKeyMaxValuesPreparedQuery(this.migrationContext.DatabaseName, this.migrationContext.OriginalTableName, &uniqueKey.Columns)
353353
if err != nil {
354354
return err
355355
}
@@ -390,7 +390,7 @@ func (this *Applier) CalculateNextIterationRangeEndValues() (hasFurtherRange boo
390390
query, explodedArgs, err := sql.BuildUniqueKeyRangeEndPreparedQuery(
391391
this.migrationContext.DatabaseName,
392392
this.migrationContext.OriginalTableName,
393-
this.migrationContext.UniqueKey.Columns.Names(),
393+
&this.migrationContext.UniqueKey.Columns,
394394
this.migrationContext.MigrationIterationRangeMinValues.AbstractValues(),
395395
this.migrationContext.MigrationRangeMaxValues.AbstractValues(),
396396
atomic.LoadInt64(&this.migrationContext.ChunkSize),
@@ -432,7 +432,7 @@ func (this *Applier) ApplyIterationInsertQuery() (chunkSize int64, rowsAffected
432432
this.migrationContext.SharedColumns.Names(),
433433
this.migrationContext.MappedSharedColumns.Names(),
434434
this.migrationContext.UniqueKey.Name,
435-
this.migrationContext.UniqueKey.Columns.Names(),
435+
&this.migrationContext.UniqueKey.Columns,
436436
this.migrationContext.MigrationIterationRangeMinValues.AbstractValues(),
437437
this.migrationContext.MigrationIterationRangeMaxValues.AbstractValues(),
438438
this.migrationContext.GetIteration() == 0,

‎go/logic/inspect.go‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -143,6 +143,7 @@ func (this *Inspector) inspectOriginalAndGhostTables() (err error) {
143143
// the `getTableColumns()` function, but it's a later patch and introduces some complexity; I feel
144144
// comfortable in doing this as a separate step.
145145
this.applyColumnTypes(this.migrationContext.DatabaseName, this.migrationContext.OriginalTableName, this.migrationContext.OriginalTableColumns, this.migrationContext.SharedColumns)
146+
this.applyColumnTypes(this.migrationContext.DatabaseName, this.migrationContext.OriginalTableName, &this.migrationContext.UniqueKey.Columns)
146147
this.applyColumnTypes(this.migrationContext.DatabaseName, this.migrationContext.GetGhostTableName(), this.migrationContext.GhostTableColumns, this.migrationContext.MappedSharedColumns)
147148

148149
for i := range this.migrationContext.SharedColumns.Columns() {
@@ -514,6 +515,11 @@ func (this *Inspector) applyColumnTypes(databaseName, tableName string, columnsL
514515
columnsList.GetColumn(columnName).Type = sql.DateTimeColumnType
515516
}
516517
}
518+
if strings.HasPrefix(columnType, "enum") {
519+
for _, columnsList := range columnsLists {
520+
columnsList.GetColumn(columnName).Type = sql.EnumColumnValue
521+
}
522+
}
517523
if charset := m.GetString("CHARACTER_SET_NAME"); charset != "" {
518524
for _, columnsList := range columnsLists {
519525
columnsList.SetCharset(columnName, charset)

‎go/sql/builder.go‎

Lines changed: 39 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -170,12 +170,12 @@ func BuildRangeComparison(columns []string, values []string, args []interface{},
170170
return result, explodedArgs, nil
171171
}
172172

173-
func BuildRangePreparedComparison(columns []string, args []interface{}, comparisonSign ValueComparisonSign) (result string, explodedArgs []interface{}, err error) {
174-
values := buildPreparedValues(len(columns))
175-
return BuildRangeComparison(columns, values, args, comparisonSign)
173+
func BuildRangePreparedComparison(columns *ColumnList, args []interface{}, comparisonSign ValueComparisonSign) (result string, explodedArgs []interface{}, err error) {
174+
values := buildColumnsPreparedValues(columns)
175+
return BuildRangeComparison(columns.Names(), values, args, comparisonSign)
176176
}
177177

178-
func BuildRangeInsertQuery(databaseName, originalTableName, ghostTableName string, sharedColumns []string, mappedSharedColumns []string, uniqueKey string, uniqueKeyColumns, rangeStartValues, rangeEndValues []string, rangeStartArgs, rangeEndArgs []interface{}, includeRangeStartValues bool, transactionalTable bool) (result string, explodedArgs []interface{}, err error) {
178+
func BuildRangeInsertQuery(databaseName, originalTableName, ghostTableName string, sharedColumns []string, mappedSharedColumns []string, uniqueKey string, uniqueKeyColumns *ColumnList, rangeStartValues, rangeEndValues []string, rangeStartArgs, rangeEndArgs []interface{}, includeRangeStartValues bool, transactionalTable bool) (result string, explodedArgs []interface{}, err error) {
179179
if len(sharedColumns) == 0 {
180180
return "", explodedArgs, fmt.Errorf("Got 0 shared columns in BuildRangeInsertQuery")
181181
}
@@ -200,12 +200,12 @@ func BuildRangeInsertQuery(databaseName, originalTableName, ghostTableName strin
200200
if includeRangeStartValues {
201201
minRangeComparisonSign = GreaterThanOrEqualsComparisonSign
202202
}
203-
rangeStartComparison, rangeExplodedArgs, err := BuildRangeComparison(uniqueKeyColumns, rangeStartValues, rangeStartArgs, minRangeComparisonSign)
203+
rangeStartComparison, rangeExplodedArgs, err := BuildRangeComparison(uniqueKeyColumns.Names(), rangeStartValues, rangeStartArgs, minRangeComparisonSign)
204204
if err != nil {
205205
return "", explodedArgs, err
206206
}
207207
explodedArgs = append(explodedArgs, rangeExplodedArgs...)
208-
rangeEndComparison, rangeExplodedArgs, err := BuildRangeComparison(uniqueKeyColumns, rangeEndValues, rangeEndArgs, LessThanOrEqualsComparisonSign)
208+
rangeEndComparison, rangeExplodedArgs, err := BuildRangeComparison(uniqueKeyColumns.Names(), rangeEndValues, rangeEndArgs, LessThanOrEqualsComparisonSign)
209209
if err != nil {
210210
return "", explodedArgs, err
211211
}
@@ -225,14 +225,14 @@ func BuildRangeInsertQuery(databaseName, originalTableName, ghostTableName strin
225225
return result, explodedArgs, nil
226226
}
227227

228-
func BuildRangeInsertPreparedQuery(databaseName, originalTableName, ghostTableName string, sharedColumns []string, mappedSharedColumns []string, uniqueKey string, uniqueKeyColumns []string, rangeStartArgs, rangeEndArgs []interface{}, includeRangeStartValues bool, transactionalTable bool) (result string, explodedArgs []interface{}, err error) {
229-
rangeStartValues := buildPreparedValues(len(uniqueKeyColumns))
230-
rangeEndValues := buildPreparedValues(len(uniqueKeyColumns))
228+
func BuildRangeInsertPreparedQuery(databaseName, originalTableName, ghostTableName string, sharedColumns []string, mappedSharedColumns []string, uniqueKey string, uniqueKeyColumns *ColumnList, rangeStartArgs, rangeEndArgs []interface{}, includeRangeStartValues bool, transactionalTable bool) (result string, explodedArgs []interface{}, err error) {
229+
rangeStartValues := buildColumnsPreparedValues(uniqueKeyColumns)
230+
rangeEndValues := buildColumnsPreparedValues(uniqueKeyColumns)
231231
return BuildRangeInsertQuery(databaseName, originalTableName, ghostTableName, sharedColumns, mappedSharedColumns, uniqueKey, uniqueKeyColumns, rangeStartValues, rangeEndValues, rangeStartArgs, rangeEndArgs, includeRangeStartValues, transactionalTable)
232232
}
233233

234-
func BuildUniqueKeyRangeEndPreparedQuery(databaseName, tableName string, uniqueKeyColumns []string, rangeStartArgs, rangeEndArgs []interface{}, chunkSize int64, includeRangeStartValues bool, hint string) (result string, explodedArgs []interface{}, err error) {
235-
if len(uniqueKeyColumns) == 0 {
234+
func BuildUniqueKeyRangeEndPreparedQuery(databaseName, tableName string, uniqueKeyColumns *ColumnList, rangeStartArgs, rangeEndArgs []interface{}, chunkSize int64, includeRangeStartValues bool, hint string) (result string, explodedArgs []interface{}, err error) {
235+
if uniqueKeyColumns.Len() == 0 {
236236
return "", explodedArgs, fmt.Errorf("Got 0 columns in BuildUniqueKeyRangeEndPreparedQuery")
237237
}
238238
databaseName = EscapeName(databaseName)
@@ -253,13 +253,18 @@ func BuildUniqueKeyRangeEndPreparedQuery(databaseName, tableName string, uniqueK
253253
}
254254
explodedArgs = append(explodedArgs, rangeExplodedArgs...)
255255

256-
uniqueKeyColumns = duplicateNames(uniqueKeyColumns)
257-
uniqueKeyColumnAscending := make([]string, len(uniqueKeyColumns), len(uniqueKeyColumns))
258-
uniqueKeyColumnDescending := make([]string, len(uniqueKeyColumns), len(uniqueKeyColumns))
259-
for i := range uniqueKeyColumns {
260-
uniqueKeyColumns[i] = EscapeName(uniqueKeyColumns[i])
261-
uniqueKeyColumnAscending[i] = fmt.Sprintf("%s asc", uniqueKeyColumns[i])
262-
uniqueKeyColumnDescending[i] = fmt.Sprintf("%s desc", uniqueKeyColumns[i])
256+
uniqueKeyColumnNames := duplicateNames(uniqueKeyColumns.Names())
257+
uniqueKeyColumnAscending := make([]string, len(uniqueKeyColumnNames), len(uniqueKeyColumnNames))
258+
uniqueKeyColumnDescending := make([]string, len(uniqueKeyColumnNames), len(uniqueKeyColumnNames))
259+
for i, column := range uniqueKeyColumns.Columns() {
260+
uniqueKeyColumnNames[i] = EscapeName(uniqueKeyColumnNames[i])
261+
if column.Type == EnumColumnValue {
262+
uniqueKeyColumnAscending[i] = fmt.Sprintf("concat(%s) asc", uniqueKeyColumnNames[i])
263+
uniqueKeyColumnDescending[i] = fmt.Sprintf("concat(%s) desc", uniqueKeyColumnNames[i])
264+
} else {
265+
uniqueKeyColumnAscending[i] = fmt.Sprintf("%s asc", uniqueKeyColumnNames[i])
266+
uniqueKeyColumnDescending[i] = fmt.Sprintf("%s desc", uniqueKeyColumnNames[i])
267+
}
263268
}
264269
result = fmt.Sprintf(`
265270
select /* gh-ost %s.%s %s */ %s
@@ -276,35 +281,39 @@ func BuildUniqueKeyRangeEndPreparedQuery(databaseName, tableName string, uniqueK
276281
order by
277282
%s
278283
limit 1
279-
`, databaseName, tableName, hint, strings.Join(uniqueKeyColumns, ", "),
280-
strings.Join(uniqueKeyColumns, ", "), databaseName, tableName,
284+
`, databaseName, tableName, hint, strings.Join(uniqueKeyColumnNames, ", "),
285+
strings.Join(uniqueKeyColumnNames, ", "), databaseName, tableName,
281286
rangeStartComparison, rangeEndComparison,
282287
strings.Join(uniqueKeyColumnAscending, ", "), chunkSize,
283288
strings.Join(uniqueKeyColumnDescending, ", "),
284289
)
285290
return result, explodedArgs, nil
286291
}
287292

288-
func BuildUniqueKeyMinValuesPreparedQuery(databaseName, tableName string, uniqueKeyColumns []string) (string, error) {
293+
func BuildUniqueKeyMinValuesPreparedQuery(databaseName, tableName string, uniqueKeyColumns *ColumnList) (string, error) {
289294
return buildUniqueKeyMinMaxValuesPreparedQuery(databaseName, tableName, uniqueKeyColumns, "asc")
290295
}
291296

292-
func BuildUniqueKeyMaxValuesPreparedQuery(databaseName, tableName string, uniqueKeyColumns []string) (string, error) {
297+
func BuildUniqueKeyMaxValuesPreparedQuery(databaseName, tableName string, uniqueKeyColumns *ColumnList) (string, error) {
293298
return buildUniqueKeyMinMaxValuesPreparedQuery(databaseName, tableName, uniqueKeyColumns, "desc")
294299
}
295300

296-
func buildUniqueKeyMinMaxValuesPreparedQuery(databaseName, tableName string, uniqueKeyColumns []string, order string) (string, error) {
297-
if len(uniqueKeyColumns) == 0 {
301+
func buildUniqueKeyMinMaxValuesPreparedQuery(databaseName, tableName string, uniqueKeyColumns *ColumnList, order string) (string, error) {
302+
if uniqueKeyColumns.Len() == 0 {
298303
return "", fmt.Errorf("Got 0 columns in BuildUniqueKeyMinMaxValuesPreparedQuery")
299304
}
300305
databaseName = EscapeName(databaseName)
301306
tableName = EscapeName(tableName)
302307

303-
uniqueKeyColumns = duplicateNames(uniqueKeyColumns)
304-
uniqueKeyColumnOrder := make([]string, len(uniqueKeyColumns), len(uniqueKeyColumns))
305-
for i := range uniqueKeyColumns {
306-
uniqueKeyColumns[i] = EscapeName(uniqueKeyColumns[i])
307-
uniqueKeyColumnOrder[i] = fmt.Sprintf("%s %s", uniqueKeyColumns[i], order)
308+
uniqueKeyColumnNames := duplicateNames(uniqueKeyColumns.Names())
309+
uniqueKeyColumnOrder := make([]string, len(uniqueKeyColumnNames), len(uniqueKeyColumnNames))
310+
for i, column := range uniqueKeyColumns.Columns() {
311+
uniqueKeyColumnNames[i] = EscapeName(uniqueKeyColumnNames[i])
312+
if column.Type == EnumColumnValue {
313+
uniqueKeyColumnOrder[i] = fmt.Sprintf("concat(%s) %s", uniqueKeyColumnNames[i], order)
314+
} else {
315+
uniqueKeyColumnOrder[i] = fmt.Sprintf("%s %s", uniqueKeyColumnNames[i], order)
316+
}
308317
}
309318
query := fmt.Sprintf(`
310319
select /* gh-ost %s.%s */ %s
@@ -313,7 +322,7 @@ func buildUniqueKeyMinMaxValuesPreparedQuery(databaseName, tableName string, uni
313322
order by
314323
%s
315324
limit 1
316-
`, databaseName, tableName, strings.Join(uniqueKeyColumns, ", "),
325+
`, databaseName, tableName, strings.Join(uniqueKeyColumnNames, ", "),
317326
databaseName, tableName,
318327
strings.Join(uniqueKeyColumnOrder, ", "),
319328
)

‎go/sql/builder_test.go‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -166,7 +166,7 @@ func TestBuildRangeInsertQuery(t *testing.T) {
166166
sharedColumns := []string{"id", "name", "position"}
167167
{
168168
uniqueKey := "PRIMARY"
169-
uniqueKeyColumns := []string{"id"}
169+
uniqueKeyColumns := NewColumnList([]string{"id"})
170170
rangeStartValues := []string{"@v1s"}
171171
rangeEndValues := []string{"@v1e"}
172172
rangeStartArgs := []interface{}{3}
@@ -185,7 +185,7 @@ func TestBuildRangeInsertQuery(t *testing.T) {
185185
}
186186
{
187187
uniqueKey := "name_position_uidx"
188-
uniqueKeyColumns := []string{"name", "position"}
188+
uniqueKeyColumns := NewColumnList([]string{"name", "position"})
189189
rangeStartValues := []string{"@v1s", "@v2s"}
190190
rangeEndValues := []string{"@v1e", "@v2e"}
191191
rangeStartArgs := []interface{}{3, 17}
@@ -212,7 +212,7 @@ func TestBuildRangeInsertQueryRenameMap(t *testing.T) {
212212
mappedSharedColumns := []string{"id", "name", "location"}
213213
{
214214
uniqueKey := "PRIMARY"
215-
uniqueKeyColumns := []string{"id"}
215+
uniqueKeyColumns := NewColumnList([]string{"id"})
216216
rangeStartValues := []string{"@v1s"}
217217
rangeEndValues := []string{"@v1e"}
218218
rangeStartArgs := []interface{}{3}
@@ -231,7 +231,7 @@ func TestBuildRangeInsertQueryRenameMap(t *testing.T) {
231231
}
232232
{
233233
uniqueKey := "name_position_uidx"
234-
uniqueKeyColumns := []string{"name", "position"}
234+
uniqueKeyColumns := NewColumnList([]string{"name", "position"})
235235
rangeStartValues := []string{"@v1s", "@v2s"}
236236
rangeEndValues := []string{"@v1e", "@v2e"}
237237
rangeStartArgs := []interface{}{3, 17}
@@ -257,7 +257,7 @@ func TestBuildRangeInsertPreparedQuery(t *testing.T) {
257257
sharedColumns := []string{"id", "name", "position"}
258258
{
259259
uniqueKey := "name_position_uidx"
260-
uniqueKeyColumns := []string{"name", "position"}
260+
uniqueKeyColumns := NewColumnList([]string{"name", "position"})
261261
rangeStartArgs := []interface{}{3, 17}
262262
rangeEndArgs := []interface{}{103, 117}
263263

@@ -279,7 +279,7 @@ func TestBuildUniqueKeyRangeEndPreparedQuery(t *testing.T) {
279279
originalTableName := "tbl"
280280
var chunkSize int64 = 500
281281
{
282-
uniqueKeyColumns := []string{"name", "position"}
282+
uniqueKeyColumns := NewColumnList([]string{"name", "position"})
283283
rangeStartArgs := []interface{}{3, 17}
284284
rangeEndArgs := []interface{}{103, 117}
285285

@@ -309,7 +309,7 @@ func TestBuildUniqueKeyRangeEndPreparedQuery(t *testing.T) {
309309
func TestBuildUniqueKeyMinValuesPreparedQuery(t *testing.T) {
310310
databaseName := "mydb"
311311
originalTableName := "tbl"
312-
uniqueKeyColumns := []string{"name", "position"}
312+
uniqueKeyColumns := NewColumnList([]string{"name", "position"})
313313
{
314314
query, err := BuildUniqueKeyMinValuesPreparedQuery(databaseName, originalTableName, uniqueKeyColumns)
315315
test.S(t).ExpectNil(err)

‎go/sql/types.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ const (
1818
UnknownColumnType ColumnType = iota
1919
TimestampColumnType = iota
2020
DateTimeColumnType = iota
21+
EnumColumnValue = iota
2122
)
2223

2324
type TimezoneConvertion struct {

‎localtests/enum-pk/create.sql‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
drop table if exists gh_ost_test;
2+
create table gh_ost_test (
3+
id int auto_increment,
4+
i int not null,
5+
e enum('red', 'green', 'blue', 'orange') null default null collate 'utf8_bin',
6+
primary key(id, e)
7+
) auto_increment=1;
8+
9+
drop event if exists gh_ost_test;
10+
delimiter ;;
11+
create event gh_ost_test
12+
on schedule every 1 second
13+
starts current_timestamp
14+
ends current_timestamp + interval 60 second
15+
on completion not preserve
16+
enable
17+
do
18+
begin
19+
insert into gh_ost_test values (null, 11, 'red');
20+
set @last_insert_id := last_insert_id();
21+
insert into gh_ost_test values (@last_insert_id, 11, 'green');
22+
insert into gh_ost_test values (null, 13, 'green');
23+
insert into gh_ost_test values (null, 17, 'blue');
24+
set @last_insert_id := last_insert_id();
25+
update gh_ost_test set e='orange' where id = @last_insert_id;
26+
insert into gh_ost_test values (null, 23, null);
27+
set @last_insert_id := last_insert_id();
28+
update gh_ost_test set i=i+1, e=null where id = @last_insert_id;
29+
end ;;

0 commit comments

Comments
 (0)