Merge pull request GoogleCloudPlatform#11088 from c2thorn/sync-main-F…

…EATURE-BRANCH-6.0.0

Sync main feature branch 6.0.0 - 7/1

.ci/gcb-push-downstream.yml

@@ -35,7 +35,7 @@ steps:
       id: tpg-sync
       waitFor: ["build-magician-binary"]
       args:
-          - wait-for-commit
+          - 'wait-for-commit'
           - 'tpg-sync'
           - $BRANCH_NAME
           - $COMMIT_SHA
@@ -46,7 +46,7 @@ steps:
       id: tpgb-sync
       waitFor: ["build-magician-binary"]
       args:
-          - wait-for-commit
+          - 'wait-for-commit'
           - 'tpgb-sync'
           - $BRANCH_NAME
           - $COMMIT_SHA
@@ -57,7 +57,7 @@ steps:
       id: tgc-sync
       waitFor: ["build-magician-binary"]
       args:
-          - wait-for-commit
+          - 'wait-for-commit'
           - 'tgc-sync'
           - $BRANCH_NAME
           - $COMMIT_SHA
@@ -68,7 +68,7 @@ steps:
       id: tf-oics-sync
       waitFor: ["build-magician-binary"]
       args:
-          - wait-for-commit
+          - 'wait-for-commit'
           - 'tf-oics-sync'
           - $BRANCH_NAME
           - $COMMIT_SHA

.ci/magician/cmd/generate_downstream.go

@@ -98,6 +98,24 @@ func execGenerateDownstream(baseBranch, command, repo, version, ref string, gh G
 		baseBranch = "main"
 	}
 
+	var syncBranchPrefix string
+	if repo == "terraform" {
+		if version == "beta" {
+			syncBranchPrefix = "tpgb-sync"
+		} else if version == "ga" {
+			syncBranchPrefix = "tpg-sync"
+		}
+	} else if repo == "terraform-google-conversion" {
+		syncBranchPrefix = "tgc-sync"
+	} else if repo == "tf-oics" {
+		syncBranchPrefix = "tf-oics-sync"
+	}
+	syncBranch := getSyncBranch(syncBranchPrefix, baseBranch)
+	if syncBranchHasCommit(ref, syncBranch, rnr) {
+		fmt.Printf("Sync branch %s already has commit %s, skipping generation\n", syncBranch, ref)
+		os.Exit(0)
+	}
+
 	mmLocalPath := filepath.Join(rnr.GetCWD(), "..", "..")
 	mmCopyPath := filepath.Join(mmLocalPath, "..", fmt.Sprintf("mm-%s-%s-%s", repo, version, command))
 	if _, err := rnr.Run("cp", []string{"-rp", mmLocalPath, mmCopyPath}, nil); err != nil {

.ci/magician/cmd/sync_branch.go

@@ -58,6 +58,11 @@ func execSyncBranchCmd(syncBranchPrefix, baseBranch, sha, githubToken string, ru
 	syncBranch := getSyncBranch(syncBranchPrefix, baseBranch)
 	fmt.Println("SYNC_BRANCH: ", syncBranch)
 
+	if syncBranchHasCommit(sha, syncBranch, runner) {
+		fmt.Printf("Commit %s already in sync branch %s, skipping sync\n", sha, syncBranch)
+		return nil
+	}
+
 	_, err := runner.Run("git", []string{"push", fmt.Sprintf("https://modular-magician:%[email protected]/GoogleCloudPlatform/magic-modules", githubToken), fmt.Sprintf("%s:%s", sha, syncBranch)}, nil)
 	return err
 }

.ci/magician/cmd/wait_for_commit.go

@@ -48,7 +48,8 @@ func execWaitForCommit(syncBranchPrefix, baseBranch, sha string, runner source.R
 	fmt.Println("SYNC_BRANCH: ", syncBranch)
 
 	if syncBranchHasCommit(sha, syncBranch, runner) {
-		return fmt.Errorf("found %s in history of %s - dying to avoid double-generating that commit", sha, syncBranch)
+		fmt.Printf("found %s in history of %s - skipping wait\n", sha, syncBranch)
+		return nil
 	}
 
 	for {
@@ -68,7 +69,7 @@ func execWaitForCommit(syncBranchPrefix, baseBranch, sha string, runner source.R
 		}
 		fmt.Println("sync branch is at: ", syncHead)
 		fmt.Println("current commit is: ", sha)
-		
+
 		if _, err := runner.Run("git", []string{"fetch", "origin", syncBranch}, nil); err != nil {
 			return err
 		}

.ci/magician/github/membership.go

@@ -82,6 +82,11 @@ var (
 			startDate: newDate(2024, 5, 22, pdtLoc),
 			endDate:   newDate(2024, 5, 28, pdtLoc),
 		},
+		{
+			id:        "melinath",
+			startDate: newDate(2024, 6, 26, pdtLoc),
+			endDate:   newDate(2024, 7, 22, pdtLoc),
+		},
 	}
 )
 

.ci/magician/vcr/tester.go

@@ -97,7 +97,8 @@ func (vt *Tester) FetchCassettes(version provider.Version, baseBranch, prNumber
 	}
 	cassettePath = filepath.Join(vt.baseDir, "cassettes", version.String())
 	vt.rnr.Mkdir(cassettePath)
-	if baseBranch != "FEATURE-BRANCH-major-release-5.0.0" {
+	if baseBranch != "FEATURE-BRANCH-major-release-6.0.0" {
+		// pull main cassettes (major release uses branch specific casssettes as primary ones)
 		bucketPath := fmt.Sprintf("gs://ci-vcr-cassettes/%sfixtures/*", version.BucketPath())
 		if err := vt.fetchBucketPath(bucketPath, cassettePath); err != nil {
 			fmt.Println("Error fetching cassettes: ", err)

docs/content/develop/breaking-changes/make-a-breaking-change.md

@@ -62,7 +62,7 @@ The general process for contributing a breaking change to the
 
 1. Make the `main` branch forwards-compatible with the major release
 2. Add deprecations and warnings to the `main` branch of `magic-modules`
-3. Add upgrade guide entries to the `main` branch of `magic-modules`
+3. Add upgrade guide entries to the `FEATURE-BRANCH-major-release-6.0.0` branch of `magic-modules`
 4. Make the breaking change on `FEATURE-BRANCH-major-release-{{% param "majorVersion" %}}`
 
 These are covered in more detail in the following sections. The upgrade guide
@@ -169,20 +169,7 @@ The deprecation message will automatically show up in the resource documentation
 Other breaking changes should be called out in the docs for the impacted field
 or resource. It is also great to log warnings at runtime if possible.
 
-### Add upgrade guide entries to the `main` branch of `magic-modules`
-
-Upgrade guide entries should be added to
-[{{< param upgradeGuide >}}](https://github.com/GoogleCloudPlatform/magic-modules/blob/main/mmv1/third_party/terraform/website/docs/guides/{{< param upgradeGuide >}}).
-Entries should focus on the changes that users need to make when upgrading
-to `{{% param "majorVersion" %}}`, rather than how to write configurations
-after upgrading.
-
-See [Terraform provider for Google Cloud 5.0.0 Upgrade Guide](https://registry.terraform.io/providers/hashicorp/google/latest/docs/guides/version_5_upgrade)
-and other upgrade guides for examples.
-
-The upgrade guide and the actual breaking change will be merged only after both are completed.
-
-### Make the breaking change on `FEATURE-BRANCH-major-release-{{% param "majorVersion" %}}`
+### Make the change on `FEATURE-BRANCH-major-release-{{% param "majorVersion" %}}`
 
 When working on your breaking change, make sure that your base branch
 is `FEATURE-BRANCH-major-release-{{% param "majorVersion" %}}`. This
@@ -207,14 +194,17 @@ with the following changes:
    are present on the major release branch. Changes to the `main` branch will be
    merged into the major release branch every Monday.
 1. Make the breaking change.
+1. Add the upgrade guide entries to
+[{{< param upgradeGuide >}}](https://github.com/GoogleCloudPlatform/magic-modules/blob/FEATURE-BRANCH-major-release-6.0.0/mmv1/third_party/terraform/website/docs/guides/{{< param upgradeGuide >}}). Entries should focus on the changes that users need to make when upgrading
+to `{{% param "majorVersion" %}}`, rather than how to write configurations
+after upgrading. See [Terraform provider for Google Cloud 5.0.0 Upgrade Guide](https://registry.terraform.io/providers/hashicorp/google/latest/docs/guides/version_5_upgrade)
+and other upgrade guides for examples.
 1. Remove any deprecation notices and warnings (including in documentation) not already removed by the breaking change.
 1. When you create your pull request,
    [change the base branch](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/changing-the-base-branch-of-a-pull-request)
    to `FEATURE-BRANCH-major-release-{{% param "majorVersion" %}}`
 1. To resolve merge conflicts with `git rebase` or `git merge`, use `FEATURE-BRANCH-major-release-{{% param "majorVersion" %}}` instead of `main`.
 
-The upgrade guide and the actual breaking change will be merged only after both are completed.
-
 ## What's next?
 
 - [Run tests]({{< ref "/develop/test/run-tests.md" >}})

mmv1/api/resource.go

@@ -16,8 +16,8 @@ import (
 	"fmt"
 	"maps"
 	"regexp"
-	"strings"
 	"sort"
+	"strings"
 
 	"github.com/GoogleCloudPlatform/magic-modules/mmv1/api/product"
 	"github.com/GoogleCloudPlatform/magic-modules/mmv1/api/resource"
@@ -593,9 +593,6 @@ func buildEffectiveLabelsField(name string, labels *Type) *Type {
 		"including the %s configured through Terraform, other clients and services.", name, name)
 
 	t := "KeyValueEffectiveLabels"
-	if name == "annotations" {
-		t = "KeyValueEffectiveAnnotations"
-	}
 
 	n := fmt.Sprintf("effective%s", strings.Title(name))
 
@@ -671,6 +668,10 @@ func getLabelsFieldNote(title string) string {
 		title, title, title)
 }
 
+func (r Resource) StateMigrationFile() string {
+	return fmt.Sprintf("templates/terraform/state_migrations/go/%s_%s.go.tmpl", google.Underscore(r.ProductMetadata.Name), google.Underscore(r.Name))
+}
+
 // ====================
 // Version-related methods
 // ====================
@@ -948,10 +949,7 @@ func ImportIdFormats(importFormat, identity []string, baseUrl string) []string {
 	var idFormats []string
 	if len(importFormat) == 0 {
 		underscoredBaseUrl := baseUrl
-		// TODO Q2: underscore base url needed?
-		// underscored_base_url = base_url.gsub(
-		//     /{{[[:word:]]+}}/, &:underscore
-		//   )
+
 		if len(identity) == 0 {
 			idFormats = []string{fmt.Sprintf("%s/{{name}}", underscoredBaseUrl)}
 		} else {
@@ -960,7 +958,7 @@ func ImportIdFormats(importFormat, identity []string, baseUrl string) []string {
 				transformedIdentity = append(transformedIdentity, fmt.Sprintf("{{%s}}", id))
 			}
 			identityPath := strings.Join(transformedIdentity, "/")
-			idFormats = []string{fmt.Sprintf("%s/%s", underscoredBaseUrl, identityPath)}
+			idFormats = []string{fmt.Sprintf("%s/%s", underscoredBaseUrl, google.Underscore(identityPath))}
 		}
 	} else {
 		idFormats = importFormat
@@ -985,15 +983,9 @@ func ImportIdFormats(importFormat, identity []string, baseUrl string) []string {
 	// `{{project}}/{{%name}}` as there is no way to differentiate between
 	// project-name/resource-name and resource-name/with-slash
 	if !strings.Contains(idFormats[0], "%") {
-		idFormats = append(idFormats, shortIdFormat, shortIdDefaultProjectFormat)
-		if shortIdDefaultProjectFormat != shortIdDefaultFormat {
-			idFormats = append(idFormats, shortIdDefaultFormat)
-		}
+		idFormats = append(idFormats, shortIdFormat, shortIdDefaultProjectFormat, shortIdDefaultFormat)
 	}
 
-	idFormats = google.Reject(slices.Compact(idFormats), func(i string) bool {
-		return i == ""
-	})
 	slices.SortFunc(idFormats, func(a, b string) int {
 		i := strings.Count(a, "/")
 		j := strings.Count(b, "/")
@@ -1003,7 +995,25 @@ func ImportIdFormats(importFormat, identity []string, baseUrl string) []string {
 		return i - j
 	})
 	slices.Reverse(idFormats)
-	return idFormats
+
+	// Remove duplicates from idFormats
+	uniq := make([]string, len(idFormats))
+	uniq[0] = idFormats[0]
+	i := 1
+	j := 1
+	for j < len(idFormats) {
+		format := idFormats[j]
+		if format != uniq[i-1] {
+			uniq[i] = format
+			i++
+		}
+		j++
+	}
+
+	uniq = google.Reject(slices.Compact(uniq), func(i string) bool {
+		return i == ""
+	})
+	return uniq
 }
 
 func (r Resource) IgnoreReadPropertiesToString(e resource.Examples) string {
@@ -1014,7 +1024,7 @@ func (r Resource) IgnoreReadPropertiesToString(e resource.Examples) string {
 		}
 	}
 	for _, tp := range e.IgnoreReadExtra {
-		props = append(props, fmt.Sprintf("\"%s\"", google.Underscore(tp)))
+		props = append(props, fmt.Sprintf("\"%s\"", tp))
 	}
 	for _, tp := range r.IgnoreReadLabelsFields(r.PropertiesWithExcluded()) {
 		props = append(props, fmt.Sprintf("\"%s\"", google.Underscore(tp)))
@@ -1379,6 +1389,9 @@ func (r Resource) GetPropertyUpdateMasksGroups(properties []*Type, maskPrefix st
 
 // Formats whitespace in the style of the old Ruby generator's descriptions in documentation
 func (r Resource) FormatDocDescription(desc string, indent bool) string {
+	if desc == "" {
+		return ""
+	}
 	returnString := desc
 	if indent {
 		returnString = strings.ReplaceAll(returnString, "\n\n", "\n")
@@ -1387,7 +1400,7 @@ func (r Resource) FormatDocDescription(desc string, indent bool) string {
 		// fix removing for ruby -> go transition diffs
 		returnString = strings.ReplaceAll(returnString, "\n  \n  **Note**: This field is non-authoritative,", "\n\n  **Note**: This field is non-authoritative,")
 
-		return strings.TrimSuffix(returnString, "\n  ")
+		return fmt.Sprintf("\n  %s", strings.TrimSuffix(returnString, "\n  "))
 	}
 	return strings.TrimSuffix(returnString, "\n")
 }
@@ -1455,8 +1468,8 @@ func (r Resource) propertiesWithCustomUpdate(properties []*Type) []*Type {
 	})
 }
 
-func (r Resource) PropertiesByCustomUpdate() map[UpdateGroup][]*Type {
-	customUpdateProps := r.propertiesWithCustomUpdate(r.RootProperties())
+func (r Resource) PropertiesByCustomUpdate(properties []*Type) map[UpdateGroup][]*Type {
+	customUpdateProps := r.propertiesWithCustomUpdate(properties)
 	groupedCustomUpdateProps := map[UpdateGroup][]*Type{}
 	for _, prop := range customUpdateProps {
 		groupedProperty := UpdateGroup{UpdateUrl: prop.UpdateUrl,
@@ -1477,21 +1490,28 @@ func (r Resource) PropertiesByCustomUpdateGroups() []UpdateGroup {
 			UpdateId:        prop.UpdateId,
 			FingerprintName: prop.FingerprintName}
 
-		if slices.Contains(updateGroups, groupedProperty){
+		if slices.Contains(updateGroups, groupedProperty) {
 			continue
 		}
 		updateGroups = append(updateGroups, groupedProperty)
 	}
-	sort.Slice(updateGroups, func(i, j int) bool { return updateGroups[i].UpdateId < updateGroups[i].UpdateId })
+	sort.Slice(updateGroups, func(i, j int) bool {
+		a := updateGroups[i]
+		b := updateGroups[j]
+		if a.UpdateVerb != b.UpdateVerb {
+			return a.UpdateVerb > b.UpdateVerb
+		}
+		return a.UpdateId < b.UpdateId
+	})
 	return updateGroups
 }
 
 func (r Resource) FieldSpecificUpdateMethods() bool {
-	return (len(r.PropertiesByCustomUpdate()) > 0)
+	return (len(r.PropertiesByCustomUpdate(r.RootProperties())) > 0)
 }
 
-func (r Resource) CustomUpdatePropertiesByKey(updateUrl string, updateId string, fingerprintName string, updateVerb string) []*Type {
-	groupedProperties := r.PropertiesByCustomUpdate()
+func (r Resource) CustomUpdatePropertiesByKey(properties []*Type, updateUrl string, updateId string, fingerprintName string, updateVerb string) []*Type {
+	groupedProperties := r.PropertiesByCustomUpdate(properties)
 	groupedProperty := UpdateGroup{UpdateUrl: updateUrl,
 		UpdateVerb:      updateVerb,
 		UpdateId:        updateId,
@@ -1526,3 +1546,11 @@ func (r Resource) VersionedProvider(exampleVersion string) bool {
 	}
 	return vp != "" && vp != "ga"
 }
+
+func (r Resource) StateUpgradersCount() []int {
+	var nums []int
+	for i := r.StateUpgradeBaseSchemaVersion; i < r.SchemaVersion; i++ {
+		nums = append(nums, i)
+	}
+	return nums
+}

mmv1/api/resource.rb

@@ -396,7 +396,11 @@ def all_nested_properties(props)
     def convert_go_file(file)
       dir, base = File.split(file)
       base.slice! '.erb'
-      "#{dir}/go/#{base}.tmpl"
+      if dir.end_with?('terraform')
+        "#{dir}/#{base}.go.tmpl"
+      else
+        "#{dir}/go/#{base}.tmpl"
+      end
     end
 
     # All settable properties in the resource.

mmv1/api/resource/docs.go

@@ -33,10 +33,10 @@ type Docs struct {
 	Note string
 
 	// attr_reader :
-	RequiredProperties string
+	RequiredProperties string `yaml:"required_properties"`
 
 	// attr_reader :
-	OptionalProperties string
+	OptionalProperties string `yaml:"optional_properties"`
 
 	// attr_reader :
 	Attributes string

mmv1/api/resource/examples.go

@@ -204,6 +204,7 @@ func (e *Examples) SetHCLText() {
 	}
 	e.TestEnvVars = docTestEnvVars
 	e.DocumentationHCLText = ExecuteTemplate(e, e.ConfigPath, true)
+	e.DocumentationHCLText = regexp.MustCompile(`\n\n$`).ReplaceAllString(e.DocumentationHCLText, "\n")
 
 	// Remove region tags
 	re1 := regexp.MustCompile(`# \[[a-zA-Z_ ]+\]\n`)