diff --git a/pkg/upgrade/upgrades/BUILD.bazel b/pkg/upgrade/upgrades/BUILD.bazel index ee29302842ce..11eb6e60d839 100644 --- a/pkg/upgrade/upgrades/BUILD.bazel +++ b/pkg/upgrade/upgrades/BUILD.bazel @@ -77,6 +77,7 @@ go_test( "role_options_migration_test.go", "sampled_stmt_diagnostics_requests_test.go", "system_privileges_test.go", + "update_invalid_column_ids_in_sequence_back_references_external_test.go", "upgrade_sequence_to_be_referenced_by_ID_external_test.go", "wait_for_del_range_in_gc_job_test.go", "wait_for_schema_changes_test.go", diff --git a/pkg/upgrade/upgrades/update_invalid_column_ids_in_sequence_back_references_external_test.go b/pkg/upgrade/upgrades/update_invalid_column_ids_in_sequence_back_references_external_test.go new file mode 100644 index 000000000000..879ac985b0ad --- /dev/null +++ b/pkg/upgrade/upgrades/update_invalid_column_ids_in_sequence_back_references_external_test.go @@ -0,0 +1,199 @@ +// Copyright 2022 The Cockroach Authors. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package upgrades_test + +import ( + "context" + "encoding/hex" + "fmt" + "testing" + + "github.com/cockroachdb/cockroach/pkg/base" + "github.com/cockroachdb/cockroach/pkg/clusterversion" + "github.com/cockroachdb/cockroach/pkg/server" + "github.com/cockroachdb/cockroach/pkg/settings/cluster" + "github.com/cockroachdb/cockroach/pkg/sql/catalog/descbuilder" + "github.com/cockroachdb/cockroach/pkg/sql/catalog/descpb" + "github.com/cockroachdb/cockroach/pkg/sql/catalog/tabledesc" + "github.com/cockroachdb/cockroach/pkg/testutils/sqlutils" + "github.com/cockroachdb/cockroach/pkg/testutils/testcluster" + "github.com/cockroachdb/cockroach/pkg/util/hlc" + "github.com/cockroachdb/cockroach/pkg/util/leaktest" + "github.com/cockroachdb/cockroach/pkg/util/log" + "github.com/cockroachdb/cockroach/pkg/util/protoutil" + "github.com/stretchr/testify/require" +) + +// TestUpgradeSeqToBeReferencedByID tests that sequence references by name will be upgraded +// to be by ID in tables or views. +func TestUpdateInvalidColumnIDsInSequenceBackReferences(t *testing.T) { + defer leaktest.AfterTest(t)() + defer log.Scope(t).Close(t) + + var ( + v0 = clusterversion.ByKey(clusterversion.UpdateInvalidColumnIDsInSequenceBackReferences - 1) + v1 = clusterversion.ByKey(clusterversion.UpdateInvalidColumnIDsInSequenceBackReferences) + ) + + type TestCase struct { + seqDescsToInject []string + tblDescsToInject []string + expectedColumnIDsInSeqBackReferencesAfterUpdate map[string][][]string + } + + /* + The hex for the descriptors to inject was created by running commands + in comment above each test case in a 21.1 binary, in which we managed to + create corrupt sequence descriptors with 0-valued column ID in its + `DependedOnBy` references. + */ + testCases := []TestCase{ + { + /* + CREATE SEQUENCE s; + CREATE TABLE tbl (i INT PRIMARY KEY); + ALTER TABLE tbl ADD COLUMN j INT DEFAULT nextval('s'); + */ + seqDescsToInject: []string{ + "0aa5020a01731834203228023a0042240a0576616c756510011a0c08011040180030005014600020003000680070007800800100480052500a077072696d61727910011800220576616c7565300140004a10080010001a00200028003000380040005a007a0408002000800100880100900100980100a20106080012001800a80100b20100ba010060006a1d0a090a0561646d696e10020a080a04726f6f7410021204726f6f741801800100880103980100b201160a077072696d61727910001a0576616c756520012801b80100c20100d201080835100018002001e2011a0801100118ffffffffffffffff7f200128003204080010003801e80100f2010408001200f801008002009202009a020a08d0b197c78df6f98517b20200b80200c0021dc80200e00200", + }, + tblDescsToInject: []string{ + "0ab4020a0374626c1835203228043a0042200a016910011a0c08011040180030005014600020003000680070007800800100423a0a016a10021a0c08011040180030005014600020012a166e65787476616c2835323a3a3a524547434c41535329300050346800700078008001004803524c0a077072696d61727910011801220169300140004a10080010001a00200028003000380040005a007a0408002000800100880100900103980100a20106080012001800a80100b20100ba010060026a1d0a090a0561646d696e10020a080a04726f6f7410021204726f6f741801800102880103980100b201170a077072696d61727910001a01691a016a200120022802b80101c20100e80100f2010408001200f801008002009202009a020a08888d94b9b0f6f98517b20200b80200c0021dc80200e00200", + }, + expectedColumnIDsInSeqBackReferencesAfterUpdate: map[string][][]string{"s": {{"[2]"}}}, + }, + { + /* + CREATE SEQUENCE s; + CREATE TABLE tbl (i INT PRIMARY KEY); + ALTER TABLE tbl ADD COLUMN j INT DEFAULT nextval('s'), ADD COLUMN k INT DEFAULT nextval('s'); + */ + seqDescsToInject: []string{ + "0aa7020a01731834203228023a0042240a0576616c756510011a0c08011040180030005014600020003000680070007800800100480052500a077072696d61727910011800220576616c7565300140004a10080010001a00200028003000380040005a007a0408002000800100880100900100980100a20106080012001800a80100b20100ba010060006a1d0a090a0561646d696e10020a080a04726f6f7410021204726f6f741801800100880103980100b201160a077072696d61727910001a0576616c756520012801b80100c20100d2010a08351000180018002001e2011a0801100118ffffffffffffffff7f200128003204080010003801e80100f2010408001200f801008002009202009a020a08e0ed9d81e6f6a08617b20200b80200c0021dc80200e00200", + }, + tblDescsToInject: []string{ + "0af5020a0374626c1835203228043a0042200a016910011a0c08011040180030005014600020003000680070007800800100423a0a016a10021a0c08011040180030005014600020012a166e65787476616c2835323a3a3a524547434c4153532930005034680070007800800100423a0a016b10031a0c08011040180030005014600020012a166e65787476616c2835323a3a3a524547434c41535329300050346800700078008001004804524c0a077072696d61727910011801220169300140004a10080010001a00200028003000380040005a007a0408002000800100880100900103980100a20106080012001800a80100b20100ba010060026a1d0a090a0561646d696e10020a080a04726f6f7410021204726f6f741801800102880103980100b2011c0a077072696d61727910001a01691a016a1a016b2001200220032802b80101c20100e80100f2010408001200f801008002009202009a020a08888e9486fef6a08617b20200b80200c0021dc80200e00200", + }, + expectedColumnIDsInSeqBackReferencesAfterUpdate: map[string][][]string{"s": {{"[2, 3]"}}}, + }, + { + /* + CREATE SEQUENCE s; + CREATE TABLE tbl_1 (i INT PRIMARY KEY); + ALTER TABLE tbl_1 ADD COLUMN j INT DEFAULT nextval('s'); + + CREATE TABLE tbl_2 (i INT PRIMARY KEY); + ALTER TABLE tbl_2 ADD COLUMN j INT DEFAULT nextval('s'), ADD COLUMN k INT DEFAULT nextval('s'); + + CREATE TABLE tbl_3 (i INT PRIMARY KEY, j INT DEFAULT nextval('s')); + ALTER TABLE tbl_3 ADD COLUMN k INT DEFAULT nextval('s'); + */ + seqDescsToInject: []string{ + "0abf020a01731834203228053a0042240a0576616c756510011a0c08011040180030005014600020003000680070007800800100480052500a077072696d61727910011800220576616c7565300140004a10080010001a00200028003000380040005a007a0408002000800100880100900100980100a20106080012001800a80100b20100ba010060006a1d0a090a0561646d696e10020a080a04726f6f7410021204726f6f741801800100880103980100b201160a077072696d61727910001a0576616c756520012801b80100c20100d201080835100018002001d2010a08361000180018002001d2010a08371000180218002001e2011a0801100118ffffffffffffffff7f200128003204080010003801e80100f2010408001200f801008002009202009a020a08b0d8cd8fe5f2a18617b20200b80200c0021dc80200e00200", + }, + tblDescsToInject: []string{ + "0ab6020a0574626c5f311835203228043a0042200a016910011a0c08011040180030005014600020003000680070007800800100423a0a016a10021a0c08011040180030005014600020012a166e65787476616c2835323a3a3a524547434c41535329300050346800700078008001004803524c0a077072696d61727910011801220169300140004a10080010001a00200028003000380040005a007a0408002000800100880100900103980100a20106080012001800a80100b20100ba010060026a1d0a090a0561646d696e10020a080a04726f6f7410021204726f6f741801800102880103980100b201170a077072696d61727910001a01691a016a200120022802b80101c20100e80100f2010408001200f801008002009202009a020a08d0c5e6ffc5f3a18617b20200b80200c0021dc80200e00200", + "0af7020a0574626c5f321836203228043a0042200a016910011a0c08011040180030005014600020003000680070007800800100423a0a016a10021a0c08011040180030005014600020012a166e65787476616c2835323a3a3a524547434c4153532930005034680070007800800100423a0a016b10031a0c08011040180030005014600020012a166e65787476616c2835323a3a3a524547434c41535329300050346800700078008001004804524c0a077072696d61727910011801220169300140004a10080010001a00200028003000380040005a007a0408002000800100880100900103980100a20106080012001800a80100b20100ba010060026a1d0a090a0561646d696e10020a080a04726f6f7410021204726f6f741801800102880103980100b2011c0a077072696d61727910001a01691a016a1a016b2001200220032802b80101c20100e80100f2010408001200f801008002009202009a020a08b8d3ffc9d4f3a18617b20200b80200c0021dc80200e00200", + "0af7020a0574626c5f331837203228043a0042200a016910011a0c08011040180030005014600020003000680070007800800100423a0a016a10021a0c08011040180030005014600020012a166e65787476616c2835323a3a3a524547434c4153532930005034680070007800800100423a0a016b10031a0c08011040180030005014600020012a166e65787476616c2835323a3a3a524547434c41535329300050346800700078008001004804524c0a077072696d61727910011801220169300140004a10080010001a00200028003000380040005a007a0408002000800100880100900103980100a20106080012001800a80100b20100ba010060026a1d0a090a0561646d696e10020a080a04726f6f7410021204726f6f741801800102880103980100b2011c0a077072696d61727910001a01691a016a1a016b2001200220032802b80101c20100e80100f2010408001200f801008002009202009a020a08c8b8f5bea1f9a18617b20200b80200c0021dc80200e00200", + }, + expectedColumnIDsInSeqBackReferencesAfterUpdate: map[string][][]string{"s": {{"[2]"}, {"[2, 3]"}, {"[2, 3]"}}}, + }, + { + /* + CREATE SEQUENCE s1; + CREATE SEQUENCE s2; + + CREATE TABLE tbl (i INT PRIMARY KEY); + ALTER TABLE tbl ADD COLUMN j INT DEFAULT nextval('s1'); + ALTER TABLE tbl ADD COLUMN k INT DEFAULT nextval('s2'); + */ + seqDescsToInject: []string{ + "0aa6020a0273311834203228023a0042240a0576616c756510011a0c08011040180030005014600020003000680070007800800100480052500a077072696d61727910011800220576616c7565300140004a10080010001a00200028003000380040005a007a0408002000800100880100900100980100a20106080012001800a80100b20100ba010060006a1d0a090a0561646d696e10020a080a04726f6f7410021204726f6f741801800100880103980100b201160a077072696d61727910001a0576616c756520012801b80100c20100d201080836100018002001e2011a0801100118ffffffffffffffff7f200128003204080010003801e80100f2010408001200f801008002009202009a020a08b8e6cacbf597f28617b20200b80200c0021dc80200e00200", + "0aa6020a0273321835203228023a0042240a0576616c756510011a0c08011040180030005014600020003000680070007800800100480052500a077072696d61727910011800220576616c7565300140004a10080010001a00200028003000380040005a007a0408002000800100880100900100980100a20106080012001800a80100b20100ba010060006a1d0a090a0561646d696e10020a080a04726f6f7410021204726f6f741801800100880103980100b201160a077072696d61727910001a0576616c756520012801b80100c20100d201080836100018002001e2011a0801100118ffffffffffffffff7f200128003204080010003801e80100f2010408001200f801008002009202009a020a08f0b2baff8498f28617b20200b80200c0021dc80200e00200", + }, + tblDescsToInject: []string{ + "0af5020a0374626c1836203228073a0042200a016910011a0c08011040180030005014600020003000680070007800800100423a0a016a10021a0c08011040180030005014600020012a166e65787476616c2835323a3a3a524547434c4153532930005034680070007800800100423a0a016b10031a0c08011040180030005014600020012a166e65787476616c2835333a3a3a524547434c41535329300050356800700078008001004804524c0a077072696d61727910011801220169300140004a10080010001a00200028003000380040005a007a0408002000800100880100900103980100a20106080012001800a80100b20100ba010060026a1d0a090a0561646d696e10020a080a04726f6f7410021204726f6f741801800103880103980100b2011c0a077072696d61727910001a01691a016a1a016b2001200220032802b80101c20100e80100f2010408001200f801008002009202009a020a08d0fc82eb819cf28617b20200b80200c0021dc80200e00200", + }, + expectedColumnIDsInSeqBackReferencesAfterUpdate: map[string][][]string{"s1": {{"[2]"}}, "s2": {{"[3]"}}}, + }, + } + + run := func(testCase TestCase) { + ctx := context.Background() + settings := cluster.MakeTestingClusterSettingsWithVersions(v1, v0, false /* initializeVersion */) + require.NoError(t, clusterversion.Initialize(ctx, v0, &settings.SV)) + + tc := testcluster.StartTestCluster(t, 1, base.TestClusterArgs{ + ServerArgs: base.TestServerArgs{ + Settings: settings, + Knobs: base.TestingKnobs{ + Server: &server.TestingKnobs{ + DisableAutomaticVersionUpgrade: make(chan struct{}), + BinaryVersionOverride: v0, + }, + }, + }, + }) + defer tc.Stopper().Stop(ctx) + + sqlDB := tc.ServerConn(0) + tdb := sqlutils.MakeSQLRunner(sqlDB) + + var parentID, parentSchemaID descpb.ID + tdb.Exec(t, "CREATE TABLE temp_tbl()") + tdb.QueryRow(t, `SELECT "parentID", "parentSchemaID" FROM system.namespace WHERE name = 'temp_tbl'`). + Scan(&parentID, &parentSchemaID) + + // A function that decode a table descriptor from a hex-encoded string and insert it into the test cluster. + decodeTableDescriptorAndInsert := func(hexEncodedDescriptor string) { + var desc descpb.Descriptor + decodedDescriptor, err := hex.DecodeString(hexEncodedDescriptor) + require.NoError(t, err) + require.NoError(t, protoutil.Unmarshal(decodedDescriptor, &desc)) + // Run post deserialization changes. + b := descbuilder.NewBuilderWithMVCCTimestamp(&desc, hlc.Timestamp{WallTime: 1}) + require.NoError(t, b.RunPostDeserializationChanges()) + // Modify this descriptor's parentID and parentSchemaID + tableDesc := b.(tabledesc.TableDescriptorBuilder).BuildCreatedMutableTable() + tableDesc.ParentID = parentID + tableDesc.UnexposedParentSchemaID = parentSchemaID + // Insert the descriptor into test cluster. + require.NoError(t, sqlutils.InjectDescriptors( + ctx, sqlDB, []*descpb.Descriptor{tableDesc.DescriptorProto()}, true, /* force */ + )) + } + + // Decode and insert the sequence descriptors and table descriptors. + for _, seq := range testCase.seqDescsToInject { + decodeTableDescriptorAndInsert(seq) + } + for _, tbl := range testCase.tblDescsToInject { + decodeTableDescriptorAndInsert(tbl) + } + + // Upgrade to the new cluster version. + tdb.Exec(t, `SET CLUSTER SETTING version = $1`, v1.String()) + tdb.CheckQueryResultsRetry(t, "SHOW CLUSTER SETTING version", + [][]string{{v1.String()}}) + + // Assert the upgrade logic correctly updated the invalid column IDs in the sequences' + // back references. + for seqName, expectedOutput := range testCase.expectedColumnIDsInSeqBackReferencesAfterUpdate { + query := fmt.Sprintf("select jsonb_array_elements(crdb_internal.pb_to_json('cockroach.sql.sqlbase.Descriptor', descriptor)->'table'->'dependedOnBy')->'columnIds' from system.descriptor where id = '%v'::REGCLASS;", seqName) + tdb.CheckQueryResults(t, query, expectedOutput) + } + } + + for _, testCase := range testCases { + run(testCase) + } + +}