diff --git a/mmv1/main.go b/mmv1/main.go index b65c7a4374af..d5904b33fdad 100644 --- a/mmv1/main.go +++ b/mmv1/main.go @@ -103,8 +103,9 @@ func GenerateProducts(product, resource, providerName, version, outputPath, base productsToGenerate = append(productsToGenerate, p.PackagePath) } } else { - var productToGenerate = fmt.Sprintf("products/%s", product) - productsToGenerate = []string{productToGenerate} + for _, prod := range strings.Split(product, ",") { + productsToGenerate = append(productsToGenerate, fmt.Sprintf("products/%s", strings.TrimSpace(prod))) + } } for _, productApi := range loadedProducts { @@ -122,8 +123,8 @@ func GenerateProducts(product, resource, providerName, version, outputPath, base }) // In order to only copy/compile files once per provider this must be called outside - // of the products loop. Create an MMv1 provider with an arbitrary product (the first loaded). - providerToGenerate := newProvider(providerName, version, productsForVersion[0], startTime, wrappedFS) + // of the products loop. Create an MMv1 provider with a nil product to trigger shared file behavior. + providerToGenerate := newProvider(providerName, version, nil, startTime, wrappedFS) providerToGenerate.CopyCommonFiles(outputPath, generateCode, generateDocs) if generateCode { @@ -148,6 +149,11 @@ func GenerateProduct(version, providerName string, productApi *api.Product, outp log.Printf("%s: Generating files", productApi.PackagePath) providerToGenerate := newProvider(providerName, version, productApi, startTime, fsys) providerToGenerate.Generate(outputPath, resourceToGenerate, generateCode, generateDocs) + + providerToGenerate.CopyCommonFiles(outputPath, generateCode, generateDocs) + if generateCode { + providerToGenerate.CompileCommonFiles(outputPath, []*api.Product{productApi}, "") + } } func newProvider(providerName, version string, productApi *api.Product, startTime time.Time, fsys fs.FS) provider.Provider { diff --git a/mmv1/provider/terraform.go b/mmv1/provider/terraform.go index 08ae23f035bd..01877b911d82 100644 --- a/mmv1/provider/terraform.go +++ b/mmv1/provider/terraform.go @@ -59,14 +59,16 @@ func NewTerraform(product *api.Product, versionName string, startTime time.Time, IAMResourceCount: 0, Product: product, TargetVersionName: versionName, - Version: *product.VersionObjOrClosest(versionName), StartTime: startTime, templateFS: templateFS, } - t.Product.ImportPath = ImportPathFromVersion(versionName) - for _, r := range t.Product.Objects { - r.ImportPath = t.Product.ImportPath + if product != nil { + t.Version = *product.VersionObjOrClosest(versionName) + t.Product.ImportPath = ImportPathFromVersion(versionName) + for _, r := range t.Product.Objects { + r.ImportPath = t.Product.ImportPath + } } return t @@ -458,6 +460,20 @@ func (t Terraform) getCommonCopyFiles(versionName string, generateCode, generate // key is the target file and value is the source file commonCopyFiles := make(map[string]string, 0) + // Case 0: If we're generating a specific product, only copy files for that product. + if t.Product != nil { + if !generateCode { + return commonCopyFiles + } + googleDir := "google" + if versionName != "ga" { + googleDir = fmt.Sprintf("google-%s", versionName) + } + files := t.getCopyFilesInFolder("third_party/terraform/services/"+t.Product.ApiName, googleDir) + maps.Copy(commonCopyFiles, files) + return commonCopyFiles + } + // Case 1: When copy all of files except .tmpl in a folder to the root directory of downstream repository, // save the folder name to foldersCopiedToRootDir foldersCopiedToRootDir := []string{"third_party/terraform/META.d", "third_party/terraform/version"} @@ -493,7 +509,6 @@ func (t Terraform) getCommonCopyFiles(versionName string, generateCode, generate "third_party/terraform/fwvalidators", "third_party/terraform/provider", "third_party/terraform/registry", - "third_party/terraform/services", "third_party/terraform/sweeper", "third_party/terraform/test-fixtures", "third_party/terraform/tpgdclresource", @@ -596,7 +611,9 @@ func (t Terraform) CopyFileList(outputFolder string, files map[string]string, ge // Compiles files that are shared at the provider level func (t Terraform) CompileCommonFiles(outputFolder string, products []*api.Product, overridePath string) { log.Printf("Generating common files for %s", ProviderName(t)) - t.generateResourcesForVersion(products) + if t.Product == nil { + t.generateResourcesForVersion(products) + } files := t.getCommonCompileFiles(t.TargetVersionName) templateData := NewTemplateData(outputFolder, t.TargetVersionName, t.templateFS) t.CompileFileList(outputFolder, files, *templateData, products) @@ -608,6 +625,14 @@ func (t Terraform) getCommonCompileFiles(versionName string) map[string]string { // key is the target file and the value is the source file commonCompileFiles := make(map[string]string, 0) + if t.Product != nil { + googleDir := "google" + if versionName != "ga" { + googleDir = fmt.Sprintf("google-%s", versionName) + } + return t.getCompileFilesInFolder("third_party/terraform/services/"+t.Product.ApiName, googleDir) + } + // Case 1: When compile all of files except .tmpl in a folder to the root directory of downstream repository, // save the folder name to foldersCopiedToRootDir foldersCompiledToRootDir := []string{"third_party/terraform/scripts"} @@ -628,7 +653,6 @@ func (t Terraform) getCommonCompileFiles(versionName string) map[string]string { "third_party/terraform/fwresource", "third_party/terraform/fwtransport", "third_party/terraform/provider", - "third_party/terraform/services", "third_party/terraform/sweeper", "third_party/terraform/test-fixtures", "third_party/terraform/tpgdclresource", diff --git a/mmv1/provider/terraform_tgc.go b/mmv1/provider/terraform_tgc.go index 8d08f416c882..ec25e4837429 100644 --- a/mmv1/provider/terraform_tgc.go +++ b/mmv1/provider/terraform_tgc.go @@ -58,8 +58,10 @@ func NewTerraformGoogleConversion(product *api.Product, versionName string, star templateFS: templateFS, } - for _, r := range t.Product.Objects { - r.ImportPath = ImportPathFromVersion(versionName) + if product != nil { + for _, r := range t.Product.Objects { + r.ImportPath = ImportPathFromVersion(versionName) + } } return t @@ -185,23 +187,7 @@ func (tgc *TerraformGoogleConversion) generateCaiIamResources(products []*api.Pr func (tgc TerraformGoogleConversion) CompileCommonFiles(outputFolder string, products []*api.Product, overridePath string) { log.Printf("Compiling common files for tgc.") - tgc.generateCaiIamResources(products) - tgc.NonDefinedTests = retrieveFullManifestOfNonDefinedTests(tgc.templateFS) - - files := retrieveFullListOfTestFiles(tgc.templateFS) - for _, file := range files { - tgc.Tests = append(tgc.Tests, strings.Split(file, ".")[0]) - } - tgc.Tests = slices.Compact(tgc.Tests) - - testSource := make(map[string]string) - for target, source := range retrieveTestSourceCodeWithLocation(tgc.templateFS, ".tmpl") { - target := strings.Replace(target, "go.tmpl", "go", 1) - testSource[target] = source - } - templateData := NewTemplateData(outputFolder, tgc.TargetVersionName, tgc.templateFS) - tgc.CompileFileList(outputFolder, testSource, *templateData, products) resourceConverters := map[string]string{ "converters/google/resources/resource_converters.go": "third_party/tgc/resource_converters.go.tmpl", @@ -220,7 +206,40 @@ func (tgc TerraformGoogleConversion) CompileCommonFiles(outputFolder string, pro "converters/google/resources/services/spanner/iam_spanner_database.go": "third_party/terraform/services/spanner/iam_spanner_database.go.tmpl", "converters/google/resources/services/spanner/iam_spanner_instance.go": "third_party/terraform/services/spanner/iam_spanner_instance.go.tmpl", } - tgc.CompileFileList(outputFolder, resourceConverters, *templateData, products) + + filteredFiles := make(map[string]string) + if tgc.Product != nil { + for target, source := range resourceConverters { + if strings.Contains(target, "/services/"+tgc.Product.ApiName+"/") { + filteredFiles[target] = source + } + } + tgc.CompileFileList(outputFolder, filteredFiles, *templateData, products) + } else { + // Shared compilation + tgc.generateCaiIamResources(products) + tgc.NonDefinedTests = retrieveFullManifestOfNonDefinedTests(tgc.templateFS) + + files := retrieveFullListOfTestFiles(tgc.templateFS) + for _, file := range files { + tgc.Tests = append(tgc.Tests, strings.Split(file, ".")[0]) + } + tgc.Tests = slices.Compact(tgc.Tests) + + testSource := make(map[string]string) + for target, source := range retrieveTestSourceCodeWithLocation(tgc.templateFS, ".tmpl") { + target := strings.Replace(target, "go.tmpl", "go", 1) + testSource[target] = source + } + tgc.CompileFileList(outputFolder, testSource, *templateData, products) + + for target, source := range resourceConverters { + if !strings.Contains(target, "/services/") { + filteredFiles[target] = source + } + } + tgc.CompileFileList(outputFolder, filteredFiles, *templateData, products) + } } func (tgc TerraformGoogleConversion) CompileFileList(outputFolder string, files map[string]string, fileTemplate TemplateData, products []*api.Product) { @@ -359,9 +378,6 @@ func (tgc TerraformGoogleConversion) CopyCommonFiles(outputFolder string, genera return } - tgc.CopyFileList(outputFolder, retrieveFullListOfTestTilesWithLocation(tgc.templateFS)) - tgc.CopyFileList(outputFolder, retrieveTestSourceCodeWithLocation(tgc.templateFS, ".go")) - resourceConverters := map[string]string{ "../caiasset/asset.go": "third_party/tgc/caiasset/asset.go", "converters/google/resources/cai/constants.go": "third_party/tgc/cai/constants.go", @@ -456,7 +472,25 @@ func (tgc TerraformGoogleConversion) CopyCommonFiles(outputFolder string, genera "tfplan2cai.go": "third_party/tgc/tfplan2cai.go", "tfplan_to_cai_test.go": "third_party/tgc/tfplan_to_cai_test.go", } - tgc.CopyFileList(outputFolder, resourceConverters) + + filteredFiles := make(map[string]string) + if tgc.Product != nil { + for target, source := range resourceConverters { + if strings.Contains(target, "/services/"+tgc.Product.ApiName+"/") { + filteredFiles[target] = source + } + } + } else { + // Shared files + tgc.CopyFileList(outputFolder, retrieveFullListOfTestTilesWithLocation(tgc.templateFS)) + tgc.CopyFileList(outputFolder, retrieveTestSourceCodeWithLocation(tgc.templateFS, ".go")) + for target, source := range resourceConverters { + if !strings.Contains(target, "/services/") { + filteredFiles[target] = source + } + } + } + tgc.CopyFileList(outputFolder, filteredFiles) } func (tgc TerraformGoogleConversion) CopyFileList(outputFolder string, files map[string]string) { diff --git a/mmv1/provider/terraform_tgc_cai2hcl.go b/mmv1/provider/terraform_tgc_cai2hcl.go index a31220f0de2a..7912466ff7e2 100644 --- a/mmv1/provider/terraform_tgc_cai2hcl.go +++ b/mmv1/provider/terraform_tgc_cai2hcl.go @@ -14,10 +14,13 @@ package provider import ( + "errors" "fmt" "io/fs" "log" "os" + "path/filepath" + "strings" "time" "github.com/GoogleCloudPlatform/magic-modules/mmv1/api" @@ -43,8 +46,10 @@ func NewCaiToTerraformConversion(product *api.Product, versionName string, start templateFS: templateFS, } - for _, r := range t.Product.Objects { - r.ImportPath = ImportPathFromVersion(versionName) + if product != nil { + for _, r := range t.Product.Objects { + r.ImportPath = ImportPathFromVersion(versionName) + } } return t @@ -66,7 +71,19 @@ func (cai2hcl CaiToTerraformConversion) CopyCommonFiles(outputFolder string, gen log.Println(fmt.Errorf("error creating output directory %v: %v", outputFolder, err)) } - if err := copy.Copy("third_party/cai2hcl", outputFolder); err != nil { - log.Println(fmt.Errorf("error copying directory %v: %v", outputFolder, err)) + if cai2hcl.Product != nil { + srcDir := filepath.Join("third_party/cai2hcl/services", cai2hcl.Product.ApiName) + dstDir := filepath.Join(outputFolder, "services", cai2hcl.Product.ApiName) + if err := copy.Copy(srcDir, dstDir); err != nil && !errors.Is(err, os.ErrNotExist) { + log.Println(fmt.Errorf("error copying service directory %v: %v", srcDir, err)) + } + } else { + if err := copy.Copy("third_party/cai2hcl", outputFolder, copy.Options{ + Skip: func(srcinfo os.FileInfo, src, dest string) (bool, error) { + return strings.Contains(src, "/services/"), nil + }, + }); err != nil { + log.Println(fmt.Errorf("error copying directory %v: %v", outputFolder, err)) + } } } diff --git a/mmv1/provider/terraform_tgc_next.go b/mmv1/provider/terraform_tgc_next.go index d4effa4d6e9f..c3181dcec152 100644 --- a/mmv1/provider/terraform_tgc_next.go +++ b/mmv1/provider/terraform_tgc_next.go @@ -75,9 +75,11 @@ func NewTerraformGoogleConversionNext(product *api.Product, versionName string, templateFS: templateFS, } - t.Product.ImportPath = ImportPathFromVersion(versionName) - for _, r := range t.Product.Objects { - r.ImportPath = ImportPathFromVersion(versionName) + if product != nil { + t.Product.ImportPath = ImportPathFromVersion(versionName) + for _, r := range t.Product.Objects { + r.ImportPath = ImportPathFromVersion(versionName) + } } return t @@ -180,8 +182,6 @@ func (tgc *TerraformGoogleConversionNext) GenerateProduct(outputFolder string) { } func (tgc TerraformGoogleConversionNext) CompileCommonFiles(outputFolder string, products []*api.Product, overridePath string) { - tgc.generateResourcesForVersion(products) - resourceConverters := map[string]string{ // common "pkg/transport/config.go": "third_party/terraform/transport/config.go.tmpl", @@ -206,7 +206,25 @@ func (tgc TerraformGoogleConversionNext) CompileCommonFiles(outputFolder string, } templateData := NewTemplateData(outputFolder, tgc.TargetVersionName, tgc.templateFS) - tgc.CompileFileList(outputFolder, resourceConverters, *templateData, products) + filteredFiles := make(map[string]string) + + if tgc.Product != nil { + for target, source := range resourceConverters { + if strings.Contains(target, "/services/"+tgc.Product.ApiName+"/") { + filteredFiles[target] = source + } + } + tgc.CompileFileList(outputFolder, filteredFiles, *templateData, products) + } else { + // Shared compilation + tgc.generateResourcesForVersion(products) + for target, source := range resourceConverters { + if !strings.Contains(target, "/services/") { + filteredFiles[target] = source + } + } + tgc.CompileFileList(outputFolder, filteredFiles, *templateData, products) + } } func (tgc TerraformGoogleConversionNext) CompileFileList(outputFolder string, files map[string]string, fileTemplate TemplateData, products []*api.Product) { @@ -249,10 +267,6 @@ func (tgc TerraformGoogleConversionNext) CopyCommonFiles(outputFolder string, ge log.Println(fmt.Errorf("error creating output directory %v: %v", outputFolder, err)) } - if err := copy.Copy("third_party/tgc_next", outputFolder); err != nil { - log.Println(fmt.Errorf("error copying directory %v: %v", outputFolder, err)) - } - resourceConverters := map[string]string{ // common "pkg/envvar/envvar_utils.go": "third_party/terraform/envvar/envvar_utils.go", @@ -284,7 +298,40 @@ func (tgc TerraformGoogleConversionNext) CopyCommonFiles(outputFolder string, ge "pkg/services/resourcemanagerv3/client.go": "third_party/terraform/services/resourcemanagerv3/client.go", "pkg/services/storage/client.go": "third_party/terraform/services/storage/client.go", } - tgc.CopyFileList(outputFolder, resourceConverters) + + filteredFiles := make(map[string]string) + + if tgc.Product != nil { + // Service-specific copying + for target, source := range resourceConverters { + if strings.Contains(target, "/services/"+tgc.Product.ApiName+"/") { + filteredFiles[target] = source + } + } + tgc.CopyFileList(outputFolder, filteredFiles) + + srcDir := filepath.Join("third_party/tgc_next/pkg/services", tgc.Product.ApiName) + dstDir := filepath.Join(outputFolder, "pkg/services", tgc.Product.ApiName) + if err := copy.Copy(srcDir, dstDir); err != nil && !errors.Is(err, os.ErrNotExist) { + log.Println(fmt.Errorf("error copying service directory %v: %v", srcDir, err)) + } + } else { + // Shared copying + if err := copy.Copy("third_party/tgc_next", outputFolder, copy.Options{ + Skip: func(srcinfo os.FileInfo, src, dest string) (bool, error) { + return strings.Contains(src, "pkg/services/"), nil + }, + }); err != nil { + log.Println(fmt.Errorf("error copying directory %v: %v", outputFolder, err)) + } + + for target, source := range resourceConverters { + if !strings.Contains(target, "/services/") { + filteredFiles[target] = source + } + } + tgc.CopyFileList(outputFolder, filteredFiles) + } } func (tgc TerraformGoogleConversionNext) CopyTfToCaiCommonFiles(outputFolder string) { diff --git a/mmv1/third_party/terraform/fwprovider/framework_provider_access_token_test.go.tmpl b/mmv1/third_party/terraform/fwprovider/framework_provider_access_token_test.go.tmpl index 0764832bd172..439f31d10a4c 100644 --- a/mmv1/third_party/terraform/fwprovider/framework_provider_access_token_test.go.tmpl +++ b/mmv1/third_party/terraform/fwprovider/framework_provider_access_token_test.go.tmpl @@ -10,7 +10,9 @@ import ( "github.com/hashicorp/terraform-plugin-testing/helper/resource" "github.com/hashicorp/terraform-provider-google/google/acctest" +{{- if ne $.TargetVersionName "ga" }} _ "github.com/hashicorp/terraform-provider-google/google/services/firebase" +{{- end }} "github.com/hashicorp/terraform-provider-google/google/envvar" ) diff --git a/mmv1/third_party/terraform/fwprovider/framework_provider_scopes_test.go.tmpl b/mmv1/third_party/terraform/fwprovider/framework_provider_scopes_test.go.tmpl index 926152e2f424..f72fb7af673a 100644 --- a/mmv1/third_party/terraform/fwprovider/framework_provider_scopes_test.go.tmpl +++ b/mmv1/third_party/terraform/fwprovider/framework_provider_scopes_test.go.tmpl @@ -9,7 +9,9 @@ import ( "github.com/hashicorp/terraform-plugin-testing/helper/resource" "github.com/hashicorp/terraform-provider-google/google/acctest" +{{- if ne $.TargetVersionName "ga" }} _ "github.com/hashicorp/terraform-provider-google/google/services/firebase" +{{- end }} "github.com/hashicorp/terraform-provider-google/google/transport" ) diff --git a/mmv1/third_party/terraform/fwprovider/framework_provider_user_project_override_test.go.tmpl b/mmv1/third_party/terraform/fwprovider/framework_provider_user_project_override_test.go.tmpl index b4fd37a01f03..cd75fd19b869 100644 --- a/mmv1/third_party/terraform/fwprovider/framework_provider_user_project_override_test.go.tmpl +++ b/mmv1/third_party/terraform/fwprovider/framework_provider_user_project_override_test.go.tmpl @@ -9,8 +9,8 @@ import ( "github.com/hashicorp/terraform-plugin-testing/helper/resource" "github.com/hashicorp/terraform-provider-google/google/acctest" - _ "github.com/hashicorp/terraform-provider-google/google/services/firebase" {{ if ne $.TargetVersionName `ga` -}} + _ "github.com/hashicorp/terraform-provider-google/google/services/firebase" "github.com/hashicorp/terraform-provider-google/google/envvar" ptu "github.com/hashicorp/terraform-provider-google/google/provider/testutils" transport_tpg "github.com/hashicorp/terraform-provider-google/google/transport"