From f59f7dfde494d9097a49cd69706b00e3577555b4 Mon Sep 17 00:00:00 2001 From: Xiang Gu Date: Tue, 16 Aug 2022 15:20:53 -0400 Subject: [PATCH] upgrades_test: Added a test for the newly added upgrade We inject a descriptor that was created in v21.1 and hence contains the buggy invalid column ID in the sequence back reference that the upgrade attempts to update. We exercise the cluster upgrade and assert that our logic indeed updates the invalid column ID to the expected column ID. --- pkg/upgrade/upgrades/BUILD.bazel | 1 + ..._sequence_back_references_external_test.go | 199 ++++++++++++++++++ 2 files changed, 200 insertions(+) create mode 100644 pkg/upgrade/upgrades/update_invalid_column_ids_in_sequence_back_references_external_test.go diff --git a/pkg/upgrade/upgrades/BUILD.bazel b/pkg/upgrade/upgrades/BUILD.bazel index ee29302842ce..11eb6e60d839 100644 --- a/pkg/upgrade/upgrades/BUILD.bazel +++ b/pkg/upgrade/upgrades/BUILD.bazel @@ -77,6 +77,7 @@ go_test( "role_options_migration_test.go", "sampled_stmt_diagnostics_requests_test.go", "system_privileges_test.go", + "update_invalid_column_ids_in_sequence_back_references_external_test.go", "upgrade_sequence_to_be_referenced_by_ID_external_test.go", "wait_for_del_range_in_gc_job_test.go", "wait_for_schema_changes_test.go", diff --git a/pkg/upgrade/upgrades/update_invalid_column_ids_in_sequence_back_references_external_test.go b/pkg/upgrade/upgrades/update_invalid_column_ids_in_sequence_back_references_external_test.go new file mode 100644 index 000000000000..879ac985b0ad --- /dev/null +++ b/pkg/upgrade/upgrades/update_invalid_column_ids_in_sequence_back_references_external_test.go @@ -0,0 +1,199 @@ +// Copyright 2022 The Cockroach Authors. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package upgrades_test + +import ( + "context" + "encoding/hex" + "fmt" + "testing" + + "github.com/cockroachdb/cockroach/pkg/base" + "github.com/cockroachdb/cockroach/pkg/clusterversion" + "github.com/cockroachdb/cockroach/pkg/server" + "github.com/cockroachdb/cockroach/pkg/settings/cluster" + "github.com/cockroachdb/cockroach/pkg/sql/catalog/descbuilder" + "github.com/cockroachdb/cockroach/pkg/sql/catalog/descpb" + "github.com/cockroachdb/cockroach/pkg/sql/catalog/tabledesc" + "github.com/cockroachdb/cockroach/pkg/testutils/sqlutils" + "github.com/cockroachdb/cockroach/pkg/testutils/testcluster" + "github.com/cockroachdb/cockroach/pkg/util/hlc" + "github.com/cockroachdb/cockroach/pkg/util/leaktest" + "github.com/cockroachdb/cockroach/pkg/util/log" + "github.com/cockroachdb/cockroach/pkg/util/protoutil" + "github.com/stretchr/testify/require" +) + +// TestUpgradeSeqToBeReferencedByID tests that sequence references by name will be upgraded +// to be by ID in tables or views. +func TestUpdateInvalidColumnIDsInSequenceBackReferences(t *testing.T) { + defer leaktest.AfterTest(t)() + defer log.Scope(t).Close(t) + + var ( + v0 = clusterversion.ByKey(clusterversion.UpdateInvalidColumnIDsInSequenceBackReferences - 1) + v1 = clusterversion.ByKey(clusterversion.UpdateInvalidColumnIDsInSequenceBackReferences) + ) + + type TestCase struct { + seqDescsToInject []string + tblDescsToInject []string + expectedColumnIDsInSeqBackReferencesAfterUpdate map[string][][]string + } + + /* + The hex for the descriptors to inject was created by running commands + in comment above each test case in a 21.1 binary, in which we managed to + create corrupt sequence descriptors with 0-valued column ID in its + `DependedOnBy` references. + */ + testCases := []TestCase{ + { + /* + CREATE SEQUENCE s; + CREATE TABLE tbl (i INT PRIMARY KEY); + ALTER TABLE tbl ADD COLUMN j INT DEFAULT nextval('s'); + */ + seqDescsToInject: []string{ + "0aa5020a01731834203228023a0042240a0576616c756510011a0c08011040180030005014600020003000680070007800800100480052500a077072696d61727910011800220576616c7565300140004a10080010001a00200028003000380040005a007a0408002000800100880100900100980100a20106080012001800a80100b20100ba010060006a1d0a090a0561646d696e10020a080a04726f6f7410021204726f6f741801800100880103980100b201160a077072696d61727910001a0576616c756520012801b80100c20100d201080835100018002001e2011a0801100118ffffffffffffffff7f200128003204080010003801e80100f2010408001200f801008002009202009a020a08d0b197c78df6f98517b20200b80200c0021dc80200e00200", + }, + tblDescsToInject: []string{ + "0ab4020a0374626c1835203228043a0042200a016910011a0c08011040180030005014600020003000680070007800800100423a0a016a10021a0c08011040180030005014600020012a166e65787476616c2835323a3a3a524547434c41535329300050346800700078008001004803524c0a077072696d61727910011801220169300140004a10080010001a00200028003000380040005a007a0408002000800100880100900103980100a20106080012001800a80100b20100ba010060026a1d0a090a0561646d696e10020a080a04726f6f7410021204726f6f741801800102880103980100b201170a077072696d61727910001a01691a016a200120022802b80101c20100e80100f2010408001200f801008002009202009a020a08888d94b9b0f6f98517b20200b80200c0021dc80200e00200", + }, + expectedColumnIDsInSeqBackReferencesAfterUpdate: map[string][][]string{"s": {{"[2]"}}}, + }, + { + /* + CREATE SEQUENCE s; + CREATE TABLE tbl (i INT PRIMARY KEY); + ALTER TABLE tbl ADD COLUMN j INT DEFAULT nextval('s'), ADD COLUMN k INT DEFAULT nextval('s'); + */ + seqDescsToInject: []string{ + "0aa7020a01731834203228023a0042240a0576616c756510011a0c08011040180030005014600020003000680070007800800100480052500a077072696d61727910011800220576616c7565300140004a10080010001a00200028003000380040005a007a0408002000800100880100900100980100a20106080012001800a80100b20100ba010060006a1d0a090a0561646d696e10020a080a04726f6f7410021204726f6f741801800100880103980100b201160a077072696d61727910001a0576616c756520012801b80100c20100d2010a08351000180018002001e2011a0801100118ffffffffffffffff7f200128003204080010003801e80100f2010408001200f801008002009202009a020a08e0ed9d81e6f6a08617b20200b80200c0021dc80200e00200", + }, + tblDescsToInject: []string{ + "0af5020a0374626c1835203228043a0042200a016910011a0c08011040180030005014600020003000680070007800800100423a0a016a10021a0c08011040180030005014600020012a166e65787476616c2835323a3a3a524547434c4153532930005034680070007800800100423a0a016b10031a0c08011040180030005014600020012a166e65787476616c2835323a3a3a524547434c41535329300050346800700078008001004804524c0a077072696d61727910011801220169300140004a10080010001a00200028003000380040005a007a0408002000800100880100900103980100a20106080012001800a80100b20100ba010060026a1d0a090a0561646d696e10020a080a04726f6f7410021204726f6f741801800102880103980100b2011c0a077072696d61727910001a01691a016a1a016b2001200220032802b80101c20100e80100f2010408001200f801008002009202009a020a08888e9486fef6a08617b20200b80200c0021dc80200e00200", + }, + expectedColumnIDsInSeqBackReferencesAfterUpdate: map[string][][]string{"s": {{"[2, 3]"}}}, + }, + { + /* + CREATE SEQUENCE s; + CREATE TABLE tbl_1 (i INT PRIMARY KEY); + ALTER TABLE tbl_1 ADD COLUMN j INT DEFAULT nextval('s'); + + CREATE TABLE tbl_2 (i INT PRIMARY KEY); + ALTER TABLE tbl_2 ADD COLUMN j INT DEFAULT nextval('s'), ADD COLUMN k INT DEFAULT nextval('s'); + + CREATE TABLE tbl_3 (i INT PRIMARY KEY, j INT DEFAULT nextval('s')); + ALTER TABLE tbl_3 ADD COLUMN k INT DEFAULT nextval('s'); + */ + seqDescsToInject: []string{ + "0abf020a01731834203228053a0042240a0576616c756510011a0c08011040180030005014600020003000680070007800800100480052500a077072696d61727910011800220576616c7565300140004a10080010001a00200028003000380040005a007a0408002000800100880100900100980100a20106080012001800a80100b20100ba010060006a1d0a090a0561646d696e10020a080a04726f6f7410021204726f6f741801800100880103980100b201160a077072696d61727910001a0576616c756520012801b80100c20100d201080835100018002001d2010a08361000180018002001d2010a08371000180218002001e2011a0801100118ffffffffffffffff7f200128003204080010003801e80100f2010408001200f801008002009202009a020a08b0d8cd8fe5f2a18617b20200b80200c0021dc80200e00200", + }, + tblDescsToInject: []string{ + "0ab6020a0574626c5f311835203228043a0042200a016910011a0c08011040180030005014600020003000680070007800800100423a0a016a10021a0c08011040180030005014600020012a166e65787476616c2835323a3a3a524547434c41535329300050346800700078008001004803524c0a077072696d61727910011801220169300140004a10080010001a00200028003000380040005a007a0408002000800100880100900103980100a20106080012001800a80100b20100ba010060026a1d0a090a0561646d696e10020a080a04726f6f7410021204726f6f741801800102880103980100b201170a077072696d61727910001a01691a016a200120022802b80101c20100e80100f2010408001200f801008002009202009a020a08d0c5e6ffc5f3a18617b20200b80200c0021dc80200e00200", + "0af7020a0574626c5f321836203228043a0042200a016910011a0c08011040180030005014600020003000680070007800800100423a0a016a10021a0c08011040180030005014600020012a166e65787476616c2835323a3a3a524547434c4153532930005034680070007800800100423a0a016b10031a0c08011040180030005014600020012a166e65787476616c2835323a3a3a524547434c41535329300050346800700078008001004804524c0a077072696d61727910011801220169300140004a10080010001a00200028003000380040005a007a0408002000800100880100900103980100a20106080012001800a80100b20100ba010060026a1d0a090a0561646d696e10020a080a04726f6f7410021204726f6f741801800102880103980100b2011c0a077072696d61727910001a01691a016a1a016b2001200220032802b80101c20100e80100f2010408001200f801008002009202009a020a08b8d3ffc9d4f3a18617b20200b80200c0021dc80200e00200", + "0af7020a0574626c5f331837203228043a0042200a016910011a0c08011040180030005014600020003000680070007800800100423a0a016a10021a0c08011040180030005014600020012a166e65787476616c2835323a3a3a524547434c4153532930005034680070007800800100423a0a016b10031a0c08011040180030005014600020012a166e65787476616c2835323a3a3a524547434c41535329300050346800700078008001004804524c0a077072696d61727910011801220169300140004a10080010001a00200028003000380040005a007a0408002000800100880100900103980100a20106080012001800a80100b20100ba010060026a1d0a090a0561646d696e10020a080a04726f6f7410021204726f6f741801800102880103980100b2011c0a077072696d61727910001a01691a016a1a016b2001200220032802b80101c20100e80100f2010408001200f801008002009202009a020a08c8b8f5bea1f9a18617b20200b80200c0021dc80200e00200", + }, + expectedColumnIDsInSeqBackReferencesAfterUpdate: map[string][][]string{"s": {{"[2]"}, {"[2, 3]"}, {"[2, 3]"}}}, + }, + { + /* + CREATE SEQUENCE s1; + CREATE SEQUENCE s2; + + CREATE TABLE tbl (i INT PRIMARY KEY); + ALTER TABLE tbl ADD COLUMN j INT DEFAULT nextval('s1'); + ALTER TABLE tbl ADD COLUMN k INT DEFAULT nextval('s2'); + */ + seqDescsToInject: []string{ + "0aa6020a0273311834203228023a0042240a0576616c756510011a0c08011040180030005014600020003000680070007800800100480052500a077072696d61727910011800220576616c7565300140004a10080010001a00200028003000380040005a007a0408002000800100880100900100980100a20106080012001800a80100b20100ba010060006a1d0a090a0561646d696e10020a080a04726f6f7410021204726f6f741801800100880103980100b201160a077072696d61727910001a0576616c756520012801b80100c20100d201080836100018002001e2011a0801100118ffffffffffffffff7f200128003204080010003801e80100f2010408001200f801008002009202009a020a08b8e6cacbf597f28617b20200b80200c0021dc80200e00200", + "0aa6020a0273321835203228023a0042240a0576616c756510011a0c08011040180030005014600020003000680070007800800100480052500a077072696d61727910011800220576616c7565300140004a10080010001a00200028003000380040005a007a0408002000800100880100900100980100a20106080012001800a80100b20100ba010060006a1d0a090a0561646d696e10020a080a04726f6f7410021204726f6f741801800100880103980100b201160a077072696d61727910001a0576616c756520012801b80100c20100d201080836100018002001e2011a0801100118ffffffffffffffff7f200128003204080010003801e80100f2010408001200f801008002009202009a020a08f0b2baff8498f28617b20200b80200c0021dc80200e00200", + }, + tblDescsToInject: []string{ + "0af5020a0374626c1836203228073a0042200a016910011a0c08011040180030005014600020003000680070007800800100423a0a016a10021a0c08011040180030005014600020012a166e65787476616c2835323a3a3a524547434c4153532930005034680070007800800100423a0a016b10031a0c08011040180030005014600020012a166e65787476616c2835333a3a3a524547434c41535329300050356800700078008001004804524c0a077072696d61727910011801220169300140004a10080010001a00200028003000380040005a007a0408002000800100880100900103980100a20106080012001800a80100b20100ba010060026a1d0a090a0561646d696e10020a080a04726f6f7410021204726f6f741801800103880103980100b2011c0a077072696d61727910001a01691a016a1a016b2001200220032802b80101c20100e80100f2010408001200f801008002009202009a020a08d0fc82eb819cf28617b20200b80200c0021dc80200e00200", + }, + expectedColumnIDsInSeqBackReferencesAfterUpdate: map[string][][]string{"s1": {{"[2]"}}, "s2": {{"[3]"}}}, + }, + } + + run := func(testCase TestCase) { + ctx := context.Background() + settings := cluster.MakeTestingClusterSettingsWithVersions(v1, v0, false /* initializeVersion */) + require.NoError(t, clusterversion.Initialize(ctx, v0, &settings.SV)) + + tc := testcluster.StartTestCluster(t, 1, base.TestClusterArgs{ + ServerArgs: base.TestServerArgs{ + Settings: settings, + Knobs: base.TestingKnobs{ + Server: &server.TestingKnobs{ + DisableAutomaticVersionUpgrade: make(chan struct{}), + BinaryVersionOverride: v0, + }, + }, + }, + }) + defer tc.Stopper().Stop(ctx) + + sqlDB := tc.ServerConn(0) + tdb := sqlutils.MakeSQLRunner(sqlDB) + + var parentID, parentSchemaID descpb.ID + tdb.Exec(t, "CREATE TABLE temp_tbl()") + tdb.QueryRow(t, `SELECT "parentID", "parentSchemaID" FROM system.namespace WHERE name = 'temp_tbl'`). + Scan(&parentID, &parentSchemaID) + + // A function that decode a table descriptor from a hex-encoded string and insert it into the test cluster. + decodeTableDescriptorAndInsert := func(hexEncodedDescriptor string) { + var desc descpb.Descriptor + decodedDescriptor, err := hex.DecodeString(hexEncodedDescriptor) + require.NoError(t, err) + require.NoError(t, protoutil.Unmarshal(decodedDescriptor, &desc)) + // Run post deserialization changes. + b := descbuilder.NewBuilderWithMVCCTimestamp(&desc, hlc.Timestamp{WallTime: 1}) + require.NoError(t, b.RunPostDeserializationChanges()) + // Modify this descriptor's parentID and parentSchemaID + tableDesc := b.(tabledesc.TableDescriptorBuilder).BuildCreatedMutableTable() + tableDesc.ParentID = parentID + tableDesc.UnexposedParentSchemaID = parentSchemaID + // Insert the descriptor into test cluster. + require.NoError(t, sqlutils.InjectDescriptors( + ctx, sqlDB, []*descpb.Descriptor{tableDesc.DescriptorProto()}, true, /* force */ + )) + } + + // Decode and insert the sequence descriptors and table descriptors. + for _, seq := range testCase.seqDescsToInject { + decodeTableDescriptorAndInsert(seq) + } + for _, tbl := range testCase.tblDescsToInject { + decodeTableDescriptorAndInsert(tbl) + } + + // Upgrade to the new cluster version. + tdb.Exec(t, `SET CLUSTER SETTING version = $1`, v1.String()) + tdb.CheckQueryResultsRetry(t, "SHOW CLUSTER SETTING version", + [][]string{{v1.String()}}) + + // Assert the upgrade logic correctly updated the invalid column IDs in the sequences' + // back references. + for seqName, expectedOutput := range testCase.expectedColumnIDsInSeqBackReferencesAfterUpdate { + query := fmt.Sprintf("select jsonb_array_elements(crdb_internal.pb_to_json('cockroach.sql.sqlbase.Descriptor', descriptor)->'table'->'dependedOnBy')->'columnIds' from system.descriptor where id = '%v'::REGCLASS;", seqName) + tdb.CheckQueryResults(t, query, expectedOutput) + } + } + + for _, testCase := range testCases { + run(testCase) + } + +}