From 456cd6de4ab1b5486dc81ce2936e0d1faecf090f Mon Sep 17 00:00:00 2001 From: Stephen Lewis Date: Thu, 25 Jun 2026 11:13:54 -0700 Subject: [PATCH 1/4] Support multiple products and only copy/compile files for those products --- mmv1/main.go | 9 ++-- mmv1/provider/provider.go | 4 +- mmv1/provider/terraform.go | 66 +++++++++++++++++++++----- mmv1/provider/terraform_oics.go | 4 +- mmv1/provider/terraform_tgc.go | 4 +- mmv1/provider/terraform_tgc_cai2hcl.go | 4 +- mmv1/provider/terraform_tgc_next.go | 4 +- 7 files changed, 69 insertions(+), 26 deletions(-) diff --git a/mmv1/main.go b/mmv1/main.go index b65c7a4374af..20a9721eff54 100644 --- a/mmv1/main.go +++ b/mmv1/main.go @@ -103,8 +103,9 @@ func GenerateProducts(product, resource, providerName, version, outputPath, base productsToGenerate = append(productsToGenerate, p.PackagePath) } } else { - var productToGenerate = fmt.Sprintf("products/%s", product) - productsToGenerate = []string{productToGenerate} + for _, prod := range strings.Split(product, ",") { + productsToGenerate = append(productsToGenerate, fmt.Sprintf("products/%s", strings.TrimSpace(prod))) + } } for _, productApi := range loadedProducts { @@ -124,10 +125,10 @@ func GenerateProducts(product, resource, providerName, version, outputPath, base // In order to only copy/compile files once per provider this must be called outside // of the products loop. Create an MMv1 provider with an arbitrary product (the first loaded). providerToGenerate := newProvider(providerName, version, productsForVersion[0], startTime, wrappedFS) - providerToGenerate.CopyCommonFiles(outputPath, generateCode, generateDocs) + providerToGenerate.CopyCommonFiles(outputPath, productsToGenerate, generateCode, generateDocs) if generateCode { - providerToGenerate.CompileCommonFiles(outputPath, productsForVersion, "") + providerToGenerate.CompileCommonFiles(outputPath, productsForVersion, productsToGenerate, "") } log.Printf("Done MM generation.") diff --git a/mmv1/provider/provider.go b/mmv1/provider/provider.go index fb5aaa1fecc3..6a6864cf504b 100644 --- a/mmv1/provider/provider.go +++ b/mmv1/provider/provider.go @@ -9,8 +9,8 @@ import ( type Provider interface { Generate(string, string, bool, bool) - CopyCommonFiles(outputFolder string, generateCode, generateDocs bool) - CompileCommonFiles(outputFolder string, products []*api.Product, overridePath string) + CopyCommonFiles(outputFolder string, productsToGenerate []string, generateCode, generateDocs bool) + CompileCommonFiles(outputFolder string, products []*api.Product, productsToGenerate []string, overridePath string) } // Shared constants and functions among the providers diff --git a/mmv1/provider/terraform.go b/mmv1/provider/terraform.go index 08ae23f035bd..241e7dee7f53 100644 --- a/mmv1/provider/terraform.go +++ b/mmv1/provider/terraform.go @@ -445,16 +445,16 @@ func (t *Terraform) FullResourceName(object api.Resource) string { return fmt.Sprintf("%s_%s", productName, google.Underscore(object.Name)) } -func (t Terraform) CopyCommonFiles(outputFolder string, generateCode, generateDocs bool) { +func (t Terraform) CopyCommonFiles(outputFolder string, productsToGenerate []string, generateCode, generateDocs bool) { log.Printf("Copying common files for %s", ProviderName(t)) - files := t.getCommonCopyFiles(t.TargetVersionName, generateCode, generateDocs) + files := t.getCommonCopyFiles(t.TargetVersionName, productsToGenerate, generateCode, generateDocs) t.CopyFileList(outputFolder, files, generateCode) } // To copy a new folder, add the folder to foldersCopiedToRootDir or foldersCopiedToGoogleDir. // To copy a file, add the file to singleFiles -func (t Terraform) getCommonCopyFiles(versionName string, generateCode, generateDocs bool) map[string]string { +func (t Terraform) getCommonCopyFiles(versionName string, productsToGenerate []string, generateCode, generateDocs bool) map[string]string { // key is the target file and value is the source file commonCopyFiles := make(map[string]string, 0) @@ -472,7 +472,7 @@ func (t Terraform) getCommonCopyFiles(versionName string, generateCode, generate foldersCopiedToRootDir = append(foldersCopiedToRootDir, "third_party/terraform/website") } for _, folder := range foldersCopiedToRootDir { - files := t.getCopyFilesInFolder(folder, ".") + files := t.getCopyFilesInFolder(folder, ".", productsToGenerate) maps.Copy(commonCopyFiles, files) } @@ -509,7 +509,7 @@ func (t Terraform) getCommonCopyFiles(versionName string, generateCode, generate } // Copy files to google(or google-beta or google-private) folder in downstream for _, folder := range foldersCopiedToGoogleDir { - files := t.getCopyFilesInFolder(folder, googleDir) + files := t.getCopyFilesInFolder(folder, googleDir, productsToGenerate) maps.Copy(commonCopyFiles, files) } @@ -525,7 +525,7 @@ func (t Terraform) getCommonCopyFiles(versionName string, generateCode, generate return commonCopyFiles } -func (t Terraform) getCopyFilesInFolder(folderPath, targetDir string) map[string]string { +func (t Terraform) getCopyFilesInFolder(folderPath, targetDir string, productsToGenerate []string) map[string]string { m := make(map[string]string, 0) fs.WalkDir(t.templateFS, folderPath, func(path string, di fs.DirEntry, err error) error { if err != nil { @@ -536,6 +536,27 @@ func (t Terraform) getCopyFilesInFolder(folderPath, targetDir string) map[string return nil } + if strings.HasPrefix(path, "third_party/terraform/services/") { + rel := strings.TrimPrefix(path, "third_party/terraform/services/") + parts := strings.Split(rel, "/") + if len(parts) > 0 { + serviceName := parts[0] + if len(productsToGenerate) > 0 { + matched := false + for _, p := range productsToGenerate { + prodName := strings.TrimPrefix(p, "products/") + if prodName == serviceName { + matched = true + break + } + } + if !matched { + return nil + } + } + } + } + fname := strings.TrimPrefix(path, "third_party/terraform/") target := fname if targetDir != "." { @@ -594,17 +615,17 @@ func (t Terraform) CopyFileList(outputFolder string, files map[string]string, ge } // Compiles files that are shared at the provider level -func (t Terraform) CompileCommonFiles(outputFolder string, products []*api.Product, overridePath string) { +func (t Terraform) CompileCommonFiles(outputFolder string, products []*api.Product, productsToGenerate []string, overridePath string) { log.Printf("Generating common files for %s", ProviderName(t)) t.generateResourcesForVersion(products) - files := t.getCommonCompileFiles(t.TargetVersionName) + files := t.getCommonCompileFiles(t.TargetVersionName, productsToGenerate) templateData := NewTemplateData(outputFolder, t.TargetVersionName, t.templateFS) t.CompileFileList(outputFolder, files, *templateData, products) } // To compile a new folder, add the folder to foldersCompiledToRootDir or foldersCompiledToGoogleDir. // To compile a file, add the file to singleFiles -func (t Terraform) getCommonCompileFiles(versionName string) map[string]string { +func (t Terraform) getCommonCompileFiles(versionName string, productsToGenerate []string) map[string]string { // key is the target file and the value is the source file commonCompileFiles := make(map[string]string, 0) @@ -612,7 +633,7 @@ func (t Terraform) getCommonCompileFiles(versionName string) map[string]string { // save the folder name to foldersCopiedToRootDir foldersCompiledToRootDir := []string{"third_party/terraform/scripts"} for _, folder := range foldersCompiledToRootDir { - files := t.getCompileFilesInFolder(folder, ".") + files := t.getCompileFilesInFolder(folder, ".", productsToGenerate) maps.Copy(commonCompileFiles, files) } @@ -642,7 +663,7 @@ func (t Terraform) getCommonCompileFiles(versionName string) map[string]string { googleDir = fmt.Sprintf("google-%s", versionName) } for _, folder := range foldersCompiledToGoogleDir { - files := t.getCompileFilesInFolder(folder, googleDir) + files := t.getCompileFilesInFolder(folder, googleDir, productsToGenerate) maps.Copy(commonCompileFiles, files) } @@ -658,13 +679,34 @@ func (t Terraform) getCommonCompileFiles(versionName string) map[string]string { return commonCompileFiles } -func (t Terraform) getCompileFilesInFolder(folderPath, targetDir string) map[string]string { +func (t Terraform) getCompileFilesInFolder(folderPath, targetDir string, productsToGenerate []string) map[string]string { m := make(map[string]string, 0) fs.WalkDir(t.templateFS, folderPath, func(path string, di fs.DirEntry, err error) error { if err != nil { return err } if !di.IsDir() && strings.HasSuffix(di.Name(), ".tmpl") { + if strings.HasPrefix(path, "third_party/terraform/services/") { + rel := strings.TrimPrefix(path, "third_party/terraform/services/") + parts := strings.Split(rel, "/") + if len(parts) > 0 { + serviceName := parts[0] + if len(productsToGenerate) > 0 { + matched := false + for _, p := range productsToGenerate { + prodName := strings.TrimPrefix(p, "products/") + if prodName == serviceName { + matched = true + break + } + } + if !matched { + return nil + } + } + } + } + fname := strings.TrimPrefix(path, "third_party/terraform/") fname = strings.TrimSuffix(fname, ".tmpl") target := fname diff --git a/mmv1/provider/terraform_oics.go b/mmv1/provider/terraform_oics.go index ef21725ab0c0..0428b6b57b54 100644 --- a/mmv1/provider/terraform_oics.go +++ b/mmv1/provider/terraform_oics.go @@ -123,10 +123,10 @@ func (toics TerraformOiCS) GenerateResource(object api.Resource, templateData Te } } -func (toics TerraformOiCS) CompileCommonFiles(outputFolder string, products []*api.Product, overridePath string) { +func (toics TerraformOiCS) CompileCommonFiles(outputFolder string, products []*api.Product, productsToGenerate []string, overridePath string) { } -func (toics TerraformOiCS) CopyCommonFiles(outputFolder string, generateCode, generateDocs bool) { +func (toics TerraformOiCS) CopyCommonFiles(outputFolder string, productsToGenerate []string, generateCode, generateDocs bool) { } diff --git a/mmv1/provider/terraform_tgc.go b/mmv1/provider/terraform_tgc.go index 8d08f416c882..23eccafc72dd 100644 --- a/mmv1/provider/terraform_tgc.go +++ b/mmv1/provider/terraform_tgc.go @@ -182,7 +182,7 @@ func (tgc *TerraformGoogleConversion) generateCaiIamResources(products []*api.Pr } } -func (tgc TerraformGoogleConversion) CompileCommonFiles(outputFolder string, products []*api.Product, overridePath string) { +func (tgc TerraformGoogleConversion) CompileCommonFiles(outputFolder string, products []*api.Product, productsToGenerate []string, overridePath string) { log.Printf("Compiling common files for tgc.") tgc.generateCaiIamResources(products) @@ -352,7 +352,7 @@ func retrieveListOfManuallyDefinedTestsFromFile(fsys fs.FS, file string) []strin return tests } -func (tgc TerraformGoogleConversion) CopyCommonFiles(outputFolder string, generateCode, generateDocs bool) { +func (tgc TerraformGoogleConversion) CopyCommonFiles(outputFolder string, productsToGenerate []string, generateCode, generateDocs bool) { log.Printf("Copying common files for tgc.") if !generateCode { diff --git a/mmv1/provider/terraform_tgc_cai2hcl.go b/mmv1/provider/terraform_tgc_cai2hcl.go index a31220f0de2a..253408bd25f7 100644 --- a/mmv1/provider/terraform_tgc_cai2hcl.go +++ b/mmv1/provider/terraform_tgc_cai2hcl.go @@ -53,10 +53,10 @@ func NewCaiToTerraformConversion(product *api.Product, versionName string, start func (cai2hcl CaiToTerraformConversion) Generate(outputFolder, resourceToGenerate string, generateCode, generateDocs bool) { } -func (cai2hcl CaiToTerraformConversion) CompileCommonFiles(outputFolder string, products []*api.Product, overridePath string) { +func (cai2hcl CaiToTerraformConversion) CompileCommonFiles(outputFolder string, products []*api.Product, productsToGenerate []string, overridePath string) { } -func (cai2hcl CaiToTerraformConversion) CopyCommonFiles(outputFolder string, generateCode, generateDocs bool) { +func (cai2hcl CaiToTerraformConversion) CopyCommonFiles(outputFolder string, productsToGenerate []string, generateCode, generateDocs bool) { if !generateCode { return } diff --git a/mmv1/provider/terraform_tgc_next.go b/mmv1/provider/terraform_tgc_next.go index d4effa4d6e9f..25935845a0a8 100644 --- a/mmv1/provider/terraform_tgc_next.go +++ b/mmv1/provider/terraform_tgc_next.go @@ -179,7 +179,7 @@ func (tgc *TerraformGoogleConversionNext) GenerateProduct(outputFolder string) { tgc.replaceImportPath(targetFolder, "product.go") } -func (tgc TerraformGoogleConversionNext) CompileCommonFiles(outputFolder string, products []*api.Product, overridePath string) { +func (tgc TerraformGoogleConversionNext) CompileCommonFiles(outputFolder string, products []*api.Product, productsToGenerate []string, overridePath string) { tgc.generateResourcesForVersion(products) resourceConverters := map[string]string{ @@ -238,7 +238,7 @@ func (tgc TerraformGoogleConversionNext) CompileFileList(outputFolder string, fi } } -func (tgc TerraformGoogleConversionNext) CopyCommonFiles(outputFolder string, generateCode, generateDocs bool) { +func (tgc TerraformGoogleConversionNext) CopyCommonFiles(outputFolder string, productsToGenerate []string, generateCode, generateDocs bool) { if !generateCode { return } From dc6c26c0d64679e972f380b44abc6be37084cf27 Mon Sep 17 00:00:00 2001 From: Stephen Lewis Date: Thu, 25 Jun 2026 12:04:36 -0700 Subject: [PATCH 2/4] Refactored to handle copy/compile during product generation --- mmv1/main.go | 13 +++- mmv1/provider/provider.go | 4 +- mmv1/provider/terraform.go | 101 ++++++++++--------------- mmv1/provider/terraform_oics.go | 4 +- mmv1/provider/terraform_tgc.go | 84 ++++++++++++++------ mmv1/provider/terraform_tgc_cai2hcl.go | 29 +++++-- mmv1/provider/terraform_tgc_next.go | 73 ++++++++++++++---- 7 files changed, 195 insertions(+), 113 deletions(-) diff --git a/mmv1/main.go b/mmv1/main.go index 20a9721eff54..d5904b33fdad 100644 --- a/mmv1/main.go +++ b/mmv1/main.go @@ -123,12 +123,12 @@ func GenerateProducts(product, resource, providerName, version, outputPath, base }) // In order to only copy/compile files once per provider this must be called outside - // of the products loop. Create an MMv1 provider with an arbitrary product (the first loaded). - providerToGenerate := newProvider(providerName, version, productsForVersion[0], startTime, wrappedFS) - providerToGenerate.CopyCommonFiles(outputPath, productsToGenerate, generateCode, generateDocs) + // of the products loop. Create an MMv1 provider with a nil product to trigger shared file behavior. + providerToGenerate := newProvider(providerName, version, nil, startTime, wrappedFS) + providerToGenerate.CopyCommonFiles(outputPath, generateCode, generateDocs) if generateCode { - providerToGenerate.CompileCommonFiles(outputPath, productsForVersion, productsToGenerate, "") + providerToGenerate.CompileCommonFiles(outputPath, productsForVersion, "") } log.Printf("Done MM generation.") @@ -149,6 +149,11 @@ func GenerateProduct(version, providerName string, productApi *api.Product, outp log.Printf("%s: Generating files", productApi.PackagePath) providerToGenerate := newProvider(providerName, version, productApi, startTime, fsys) providerToGenerate.Generate(outputPath, resourceToGenerate, generateCode, generateDocs) + + providerToGenerate.CopyCommonFiles(outputPath, generateCode, generateDocs) + if generateCode { + providerToGenerate.CompileCommonFiles(outputPath, []*api.Product{productApi}, "") + } } func newProvider(providerName, version string, productApi *api.Product, startTime time.Time, fsys fs.FS) provider.Provider { diff --git a/mmv1/provider/provider.go b/mmv1/provider/provider.go index 6a6864cf504b..fb5aaa1fecc3 100644 --- a/mmv1/provider/provider.go +++ b/mmv1/provider/provider.go @@ -9,8 +9,8 @@ import ( type Provider interface { Generate(string, string, bool, bool) - CopyCommonFiles(outputFolder string, productsToGenerate []string, generateCode, generateDocs bool) - CompileCommonFiles(outputFolder string, products []*api.Product, productsToGenerate []string, overridePath string) + CopyCommonFiles(outputFolder string, generateCode, generateDocs bool) + CompileCommonFiles(outputFolder string, products []*api.Product, overridePath string) } // Shared constants and functions among the providers diff --git a/mmv1/provider/terraform.go b/mmv1/provider/terraform.go index 241e7dee7f53..fc8a72daaa8f 100644 --- a/mmv1/provider/terraform.go +++ b/mmv1/provider/terraform.go @@ -59,14 +59,16 @@ func NewTerraform(product *api.Product, versionName string, startTime time.Time, IAMResourceCount: 0, Product: product, TargetVersionName: versionName, - Version: *product.VersionObjOrClosest(versionName), StartTime: startTime, templateFS: templateFS, } - t.Product.ImportPath = ImportPathFromVersion(versionName) - for _, r := range t.Product.Objects { - r.ImportPath = t.Product.ImportPath + if product != nil { + t.Version = *product.VersionObjOrClosest(versionName) + t.Product.ImportPath = ImportPathFromVersion(versionName) + for _, r := range t.Product.Objects { + r.ImportPath = t.Product.ImportPath + } } return t @@ -445,19 +447,30 @@ func (t *Terraform) FullResourceName(object api.Resource) string { return fmt.Sprintf("%s_%s", productName, google.Underscore(object.Name)) } -func (t Terraform) CopyCommonFiles(outputFolder string, productsToGenerate []string, generateCode, generateDocs bool) { +func (t Terraform) CopyCommonFiles(outputFolder string, generateCode, generateDocs bool) { log.Printf("Copying common files for %s", ProviderName(t)) - files := t.getCommonCopyFiles(t.TargetVersionName, productsToGenerate, generateCode, generateDocs) + files := t.getCommonCopyFiles(t.TargetVersionName, generateCode, generateDocs) t.CopyFileList(outputFolder, files, generateCode) } // To copy a new folder, add the folder to foldersCopiedToRootDir or foldersCopiedToGoogleDir. // To copy a file, add the file to singleFiles -func (t Terraform) getCommonCopyFiles(versionName string, productsToGenerate []string, generateCode, generateDocs bool) map[string]string { +func (t Terraform) getCommonCopyFiles(versionName string, generateCode, generateDocs bool) map[string]string { // key is the target file and value is the source file commonCopyFiles := make(map[string]string, 0) + // Case 0: If we're generating a specific product, only copy files for that product. + if t.Product != nil { + googleDir := "google" + if versionName != "ga" { + googleDir = fmt.Sprintf("google-%s", versionName) + } + files := t.getCopyFilesInFolder("third_party/terraform/services/"+t.Product.ApiName, googleDir) + maps.Copy(commonCopyFiles, files) + return commonCopyFiles + } + // Case 1: When copy all of files except .tmpl in a folder to the root directory of downstream repository, // save the folder name to foldersCopiedToRootDir foldersCopiedToRootDir := []string{"third_party/terraform/META.d", "third_party/terraform/version"} @@ -472,7 +485,7 @@ func (t Terraform) getCommonCopyFiles(versionName string, productsToGenerate []s foldersCopiedToRootDir = append(foldersCopiedToRootDir, "third_party/terraform/website") } for _, folder := range foldersCopiedToRootDir { - files := t.getCopyFilesInFolder(folder, ".", productsToGenerate) + files := t.getCopyFilesInFolder(folder, ".") maps.Copy(commonCopyFiles, files) } @@ -493,7 +506,6 @@ func (t Terraform) getCommonCopyFiles(versionName string, productsToGenerate []s "third_party/terraform/fwvalidators", "third_party/terraform/provider", "third_party/terraform/registry", - "third_party/terraform/services", "third_party/terraform/sweeper", "third_party/terraform/test-fixtures", "third_party/terraform/tpgdclresource", @@ -509,7 +521,7 @@ func (t Terraform) getCommonCopyFiles(versionName string, productsToGenerate []s } // Copy files to google(or google-beta or google-private) folder in downstream for _, folder := range foldersCopiedToGoogleDir { - files := t.getCopyFilesInFolder(folder, googleDir, productsToGenerate) + files := t.getCopyFilesInFolder(folder, googleDir) maps.Copy(commonCopyFiles, files) } @@ -525,7 +537,7 @@ func (t Terraform) getCommonCopyFiles(versionName string, productsToGenerate []s return commonCopyFiles } -func (t Terraform) getCopyFilesInFolder(folderPath, targetDir string, productsToGenerate []string) map[string]string { +func (t Terraform) getCopyFilesInFolder(folderPath, targetDir string) map[string]string { m := make(map[string]string, 0) fs.WalkDir(t.templateFS, folderPath, func(path string, di fs.DirEntry, err error) error { if err != nil { @@ -536,27 +548,6 @@ func (t Terraform) getCopyFilesInFolder(folderPath, targetDir string, productsTo return nil } - if strings.HasPrefix(path, "third_party/terraform/services/") { - rel := strings.TrimPrefix(path, "third_party/terraform/services/") - parts := strings.Split(rel, "/") - if len(parts) > 0 { - serviceName := parts[0] - if len(productsToGenerate) > 0 { - matched := false - for _, p := range productsToGenerate { - prodName := strings.TrimPrefix(p, "products/") - if prodName == serviceName { - matched = true - break - } - } - if !matched { - return nil - } - } - } - } - fname := strings.TrimPrefix(path, "third_party/terraform/") target := fname if targetDir != "." { @@ -615,25 +606,35 @@ func (t Terraform) CopyFileList(outputFolder string, files map[string]string, ge } // Compiles files that are shared at the provider level -func (t Terraform) CompileCommonFiles(outputFolder string, products []*api.Product, productsToGenerate []string, overridePath string) { +func (t Terraform) CompileCommonFiles(outputFolder string, products []*api.Product, overridePath string) { log.Printf("Generating common files for %s", ProviderName(t)) - t.generateResourcesForVersion(products) - files := t.getCommonCompileFiles(t.TargetVersionName, productsToGenerate) + if t.Product == nil { + t.generateResourcesForVersion(products) + } + files := t.getCommonCompileFiles(t.TargetVersionName) templateData := NewTemplateData(outputFolder, t.TargetVersionName, t.templateFS) t.CompileFileList(outputFolder, files, *templateData, products) } // To compile a new folder, add the folder to foldersCompiledToRootDir or foldersCompiledToGoogleDir. // To compile a file, add the file to singleFiles -func (t Terraform) getCommonCompileFiles(versionName string, productsToGenerate []string) map[string]string { +func (t Terraform) getCommonCompileFiles(versionName string) map[string]string { // key is the target file and the value is the source file commonCompileFiles := make(map[string]string, 0) + if t.Product != nil { + googleDir := "google" + if versionName != "ga" { + googleDir = fmt.Sprintf("google-%s", versionName) + } + return t.getCompileFilesInFolder("third_party/terraform/services/"+t.Product.ApiName, googleDir) + } + // Case 1: When compile all of files except .tmpl in a folder to the root directory of downstream repository, // save the folder name to foldersCopiedToRootDir foldersCompiledToRootDir := []string{"third_party/terraform/scripts"} for _, folder := range foldersCompiledToRootDir { - files := t.getCompileFilesInFolder(folder, ".", productsToGenerate) + files := t.getCompileFilesInFolder(folder, ".") maps.Copy(commonCompileFiles, files) } @@ -649,7 +650,6 @@ func (t Terraform) getCommonCompileFiles(versionName string, productsToGenerate "third_party/terraform/fwresource", "third_party/terraform/fwtransport", "third_party/terraform/provider", - "third_party/terraform/services", "third_party/terraform/sweeper", "third_party/terraform/test-fixtures", "third_party/terraform/tpgdclresource", @@ -663,7 +663,7 @@ func (t Terraform) getCommonCompileFiles(versionName string, productsToGenerate googleDir = fmt.Sprintf("google-%s", versionName) } for _, folder := range foldersCompiledToGoogleDir { - files := t.getCompileFilesInFolder(folder, googleDir, productsToGenerate) + files := t.getCompileFilesInFolder(folder, googleDir) maps.Copy(commonCompileFiles, files) } @@ -679,34 +679,13 @@ func (t Terraform) getCommonCompileFiles(versionName string, productsToGenerate return commonCompileFiles } -func (t Terraform) getCompileFilesInFolder(folderPath, targetDir string, productsToGenerate []string) map[string]string { +func (t Terraform) getCompileFilesInFolder(folderPath, targetDir string) map[string]string { m := make(map[string]string, 0) fs.WalkDir(t.templateFS, folderPath, func(path string, di fs.DirEntry, err error) error { if err != nil { return err } if !di.IsDir() && strings.HasSuffix(di.Name(), ".tmpl") { - if strings.HasPrefix(path, "third_party/terraform/services/") { - rel := strings.TrimPrefix(path, "third_party/terraform/services/") - parts := strings.Split(rel, "/") - if len(parts) > 0 { - serviceName := parts[0] - if len(productsToGenerate) > 0 { - matched := false - for _, p := range productsToGenerate { - prodName := strings.TrimPrefix(p, "products/") - if prodName == serviceName { - matched = true - break - } - } - if !matched { - return nil - } - } - } - } - fname := strings.TrimPrefix(path, "third_party/terraform/") fname = strings.TrimSuffix(fname, ".tmpl") target := fname diff --git a/mmv1/provider/terraform_oics.go b/mmv1/provider/terraform_oics.go index 0428b6b57b54..ef21725ab0c0 100644 --- a/mmv1/provider/terraform_oics.go +++ b/mmv1/provider/terraform_oics.go @@ -123,10 +123,10 @@ func (toics TerraformOiCS) GenerateResource(object api.Resource, templateData Te } } -func (toics TerraformOiCS) CompileCommonFiles(outputFolder string, products []*api.Product, productsToGenerate []string, overridePath string) { +func (toics TerraformOiCS) CompileCommonFiles(outputFolder string, products []*api.Product, overridePath string) { } -func (toics TerraformOiCS) CopyCommonFiles(outputFolder string, productsToGenerate []string, generateCode, generateDocs bool) { +func (toics TerraformOiCS) CopyCommonFiles(outputFolder string, generateCode, generateDocs bool) { } diff --git a/mmv1/provider/terraform_tgc.go b/mmv1/provider/terraform_tgc.go index 23eccafc72dd..ec25e4837429 100644 --- a/mmv1/provider/terraform_tgc.go +++ b/mmv1/provider/terraform_tgc.go @@ -58,8 +58,10 @@ func NewTerraformGoogleConversion(product *api.Product, versionName string, star templateFS: templateFS, } - for _, r := range t.Product.Objects { - r.ImportPath = ImportPathFromVersion(versionName) + if product != nil { + for _, r := range t.Product.Objects { + r.ImportPath = ImportPathFromVersion(versionName) + } } return t @@ -182,26 +184,10 @@ func (tgc *TerraformGoogleConversion) generateCaiIamResources(products []*api.Pr } } -func (tgc TerraformGoogleConversion) CompileCommonFiles(outputFolder string, products []*api.Product, productsToGenerate []string, overridePath string) { +func (tgc TerraformGoogleConversion) CompileCommonFiles(outputFolder string, products []*api.Product, overridePath string) { log.Printf("Compiling common files for tgc.") - tgc.generateCaiIamResources(products) - tgc.NonDefinedTests = retrieveFullManifestOfNonDefinedTests(tgc.templateFS) - - files := retrieveFullListOfTestFiles(tgc.templateFS) - for _, file := range files { - tgc.Tests = append(tgc.Tests, strings.Split(file, ".")[0]) - } - tgc.Tests = slices.Compact(tgc.Tests) - - testSource := make(map[string]string) - for target, source := range retrieveTestSourceCodeWithLocation(tgc.templateFS, ".tmpl") { - target := strings.Replace(target, "go.tmpl", "go", 1) - testSource[target] = source - } - templateData := NewTemplateData(outputFolder, tgc.TargetVersionName, tgc.templateFS) - tgc.CompileFileList(outputFolder, testSource, *templateData, products) resourceConverters := map[string]string{ "converters/google/resources/resource_converters.go": "third_party/tgc/resource_converters.go.tmpl", @@ -220,7 +206,40 @@ func (tgc TerraformGoogleConversion) CompileCommonFiles(outputFolder string, pro "converters/google/resources/services/spanner/iam_spanner_database.go": "third_party/terraform/services/spanner/iam_spanner_database.go.tmpl", "converters/google/resources/services/spanner/iam_spanner_instance.go": "third_party/terraform/services/spanner/iam_spanner_instance.go.tmpl", } - tgc.CompileFileList(outputFolder, resourceConverters, *templateData, products) + + filteredFiles := make(map[string]string) + if tgc.Product != nil { + for target, source := range resourceConverters { + if strings.Contains(target, "/services/"+tgc.Product.ApiName+"/") { + filteredFiles[target] = source + } + } + tgc.CompileFileList(outputFolder, filteredFiles, *templateData, products) + } else { + // Shared compilation + tgc.generateCaiIamResources(products) + tgc.NonDefinedTests = retrieveFullManifestOfNonDefinedTests(tgc.templateFS) + + files := retrieveFullListOfTestFiles(tgc.templateFS) + for _, file := range files { + tgc.Tests = append(tgc.Tests, strings.Split(file, ".")[0]) + } + tgc.Tests = slices.Compact(tgc.Tests) + + testSource := make(map[string]string) + for target, source := range retrieveTestSourceCodeWithLocation(tgc.templateFS, ".tmpl") { + target := strings.Replace(target, "go.tmpl", "go", 1) + testSource[target] = source + } + tgc.CompileFileList(outputFolder, testSource, *templateData, products) + + for target, source := range resourceConverters { + if !strings.Contains(target, "/services/") { + filteredFiles[target] = source + } + } + tgc.CompileFileList(outputFolder, filteredFiles, *templateData, products) + } } func (tgc TerraformGoogleConversion) CompileFileList(outputFolder string, files map[string]string, fileTemplate TemplateData, products []*api.Product) { @@ -352,16 +371,13 @@ func retrieveListOfManuallyDefinedTestsFromFile(fsys fs.FS, file string) []strin return tests } -func (tgc TerraformGoogleConversion) CopyCommonFiles(outputFolder string, productsToGenerate []string, generateCode, generateDocs bool) { +func (tgc TerraformGoogleConversion) CopyCommonFiles(outputFolder string, generateCode, generateDocs bool) { log.Printf("Copying common files for tgc.") if !generateCode { return } - tgc.CopyFileList(outputFolder, retrieveFullListOfTestTilesWithLocation(tgc.templateFS)) - tgc.CopyFileList(outputFolder, retrieveTestSourceCodeWithLocation(tgc.templateFS, ".go")) - resourceConverters := map[string]string{ "../caiasset/asset.go": "third_party/tgc/caiasset/asset.go", "converters/google/resources/cai/constants.go": "third_party/tgc/cai/constants.go", @@ -456,7 +472,25 @@ func (tgc TerraformGoogleConversion) CopyCommonFiles(outputFolder string, produc "tfplan2cai.go": "third_party/tgc/tfplan2cai.go", "tfplan_to_cai_test.go": "third_party/tgc/tfplan_to_cai_test.go", } - tgc.CopyFileList(outputFolder, resourceConverters) + + filteredFiles := make(map[string]string) + if tgc.Product != nil { + for target, source := range resourceConverters { + if strings.Contains(target, "/services/"+tgc.Product.ApiName+"/") { + filteredFiles[target] = source + } + } + } else { + // Shared files + tgc.CopyFileList(outputFolder, retrieveFullListOfTestTilesWithLocation(tgc.templateFS)) + tgc.CopyFileList(outputFolder, retrieveTestSourceCodeWithLocation(tgc.templateFS, ".go")) + for target, source := range resourceConverters { + if !strings.Contains(target, "/services/") { + filteredFiles[target] = source + } + } + } + tgc.CopyFileList(outputFolder, filteredFiles) } func (tgc TerraformGoogleConversion) CopyFileList(outputFolder string, files map[string]string) { diff --git a/mmv1/provider/terraform_tgc_cai2hcl.go b/mmv1/provider/terraform_tgc_cai2hcl.go index 253408bd25f7..7912466ff7e2 100644 --- a/mmv1/provider/terraform_tgc_cai2hcl.go +++ b/mmv1/provider/terraform_tgc_cai2hcl.go @@ -14,10 +14,13 @@ package provider import ( + "errors" "fmt" "io/fs" "log" "os" + "path/filepath" + "strings" "time" "github.com/GoogleCloudPlatform/magic-modules/mmv1/api" @@ -43,8 +46,10 @@ func NewCaiToTerraformConversion(product *api.Product, versionName string, start templateFS: templateFS, } - for _, r := range t.Product.Objects { - r.ImportPath = ImportPathFromVersion(versionName) + if product != nil { + for _, r := range t.Product.Objects { + r.ImportPath = ImportPathFromVersion(versionName) + } } return t @@ -53,10 +58,10 @@ func NewCaiToTerraformConversion(product *api.Product, versionName string, start func (cai2hcl CaiToTerraformConversion) Generate(outputFolder, resourceToGenerate string, generateCode, generateDocs bool) { } -func (cai2hcl CaiToTerraformConversion) CompileCommonFiles(outputFolder string, products []*api.Product, productsToGenerate []string, overridePath string) { +func (cai2hcl CaiToTerraformConversion) CompileCommonFiles(outputFolder string, products []*api.Product, overridePath string) { } -func (cai2hcl CaiToTerraformConversion) CopyCommonFiles(outputFolder string, productsToGenerate []string, generateCode, generateDocs bool) { +func (cai2hcl CaiToTerraformConversion) CopyCommonFiles(outputFolder string, generateCode, generateDocs bool) { if !generateCode { return } @@ -66,7 +71,19 @@ func (cai2hcl CaiToTerraformConversion) CopyCommonFiles(outputFolder string, pro log.Println(fmt.Errorf("error creating output directory %v: %v", outputFolder, err)) } - if err := copy.Copy("third_party/cai2hcl", outputFolder); err != nil { - log.Println(fmt.Errorf("error copying directory %v: %v", outputFolder, err)) + if cai2hcl.Product != nil { + srcDir := filepath.Join("third_party/cai2hcl/services", cai2hcl.Product.ApiName) + dstDir := filepath.Join(outputFolder, "services", cai2hcl.Product.ApiName) + if err := copy.Copy(srcDir, dstDir); err != nil && !errors.Is(err, os.ErrNotExist) { + log.Println(fmt.Errorf("error copying service directory %v: %v", srcDir, err)) + } + } else { + if err := copy.Copy("third_party/cai2hcl", outputFolder, copy.Options{ + Skip: func(srcinfo os.FileInfo, src, dest string) (bool, error) { + return strings.Contains(src, "/services/"), nil + }, + }); err != nil { + log.Println(fmt.Errorf("error copying directory %v: %v", outputFolder, err)) + } } } diff --git a/mmv1/provider/terraform_tgc_next.go b/mmv1/provider/terraform_tgc_next.go index 25935845a0a8..c3181dcec152 100644 --- a/mmv1/provider/terraform_tgc_next.go +++ b/mmv1/provider/terraform_tgc_next.go @@ -75,9 +75,11 @@ func NewTerraformGoogleConversionNext(product *api.Product, versionName string, templateFS: templateFS, } - t.Product.ImportPath = ImportPathFromVersion(versionName) - for _, r := range t.Product.Objects { - r.ImportPath = ImportPathFromVersion(versionName) + if product != nil { + t.Product.ImportPath = ImportPathFromVersion(versionName) + for _, r := range t.Product.Objects { + r.ImportPath = ImportPathFromVersion(versionName) + } } return t @@ -179,9 +181,7 @@ func (tgc *TerraformGoogleConversionNext) GenerateProduct(outputFolder string) { tgc.replaceImportPath(targetFolder, "product.go") } -func (tgc TerraformGoogleConversionNext) CompileCommonFiles(outputFolder string, products []*api.Product, productsToGenerate []string, overridePath string) { - tgc.generateResourcesForVersion(products) - +func (tgc TerraformGoogleConversionNext) CompileCommonFiles(outputFolder string, products []*api.Product, overridePath string) { resourceConverters := map[string]string{ // common "pkg/transport/config.go": "third_party/terraform/transport/config.go.tmpl", @@ -206,7 +206,25 @@ func (tgc TerraformGoogleConversionNext) CompileCommonFiles(outputFolder string, } templateData := NewTemplateData(outputFolder, tgc.TargetVersionName, tgc.templateFS) - tgc.CompileFileList(outputFolder, resourceConverters, *templateData, products) + filteredFiles := make(map[string]string) + + if tgc.Product != nil { + for target, source := range resourceConverters { + if strings.Contains(target, "/services/"+tgc.Product.ApiName+"/") { + filteredFiles[target] = source + } + } + tgc.CompileFileList(outputFolder, filteredFiles, *templateData, products) + } else { + // Shared compilation + tgc.generateResourcesForVersion(products) + for target, source := range resourceConverters { + if !strings.Contains(target, "/services/") { + filteredFiles[target] = source + } + } + tgc.CompileFileList(outputFolder, filteredFiles, *templateData, products) + } } func (tgc TerraformGoogleConversionNext) CompileFileList(outputFolder string, files map[string]string, fileTemplate TemplateData, products []*api.Product) { @@ -238,7 +256,7 @@ func (tgc TerraformGoogleConversionNext) CompileFileList(outputFolder string, fi } } -func (tgc TerraformGoogleConversionNext) CopyCommonFiles(outputFolder string, productsToGenerate []string, generateCode, generateDocs bool) { +func (tgc TerraformGoogleConversionNext) CopyCommonFiles(outputFolder string, generateCode, generateDocs bool) { if !generateCode { return } @@ -249,10 +267,6 @@ func (tgc TerraformGoogleConversionNext) CopyCommonFiles(outputFolder string, pr log.Println(fmt.Errorf("error creating output directory %v: %v", outputFolder, err)) } - if err := copy.Copy("third_party/tgc_next", outputFolder); err != nil { - log.Println(fmt.Errorf("error copying directory %v: %v", outputFolder, err)) - } - resourceConverters := map[string]string{ // common "pkg/envvar/envvar_utils.go": "third_party/terraform/envvar/envvar_utils.go", @@ -284,7 +298,40 @@ func (tgc TerraformGoogleConversionNext) CopyCommonFiles(outputFolder string, pr "pkg/services/resourcemanagerv3/client.go": "third_party/terraform/services/resourcemanagerv3/client.go", "pkg/services/storage/client.go": "third_party/terraform/services/storage/client.go", } - tgc.CopyFileList(outputFolder, resourceConverters) + + filteredFiles := make(map[string]string) + + if tgc.Product != nil { + // Service-specific copying + for target, source := range resourceConverters { + if strings.Contains(target, "/services/"+tgc.Product.ApiName+"/") { + filteredFiles[target] = source + } + } + tgc.CopyFileList(outputFolder, filteredFiles) + + srcDir := filepath.Join("third_party/tgc_next/pkg/services", tgc.Product.ApiName) + dstDir := filepath.Join(outputFolder, "pkg/services", tgc.Product.ApiName) + if err := copy.Copy(srcDir, dstDir); err != nil && !errors.Is(err, os.ErrNotExist) { + log.Println(fmt.Errorf("error copying service directory %v: %v", srcDir, err)) + } + } else { + // Shared copying + if err := copy.Copy("third_party/tgc_next", outputFolder, copy.Options{ + Skip: func(srcinfo os.FileInfo, src, dest string) (bool, error) { + return strings.Contains(src, "pkg/services/"), nil + }, + }); err != nil { + log.Println(fmt.Errorf("error copying directory %v: %v", outputFolder, err)) + } + + for target, source := range resourceConverters { + if !strings.Contains(target, "/services/") { + filteredFiles[target] = source + } + } + tgc.CopyFileList(outputFolder, filteredFiles) + } } func (tgc TerraformGoogleConversionNext) CopyTfToCaiCommonFiles(outputFolder string) { From 89cbe0526644b5eaa73acb4a094b8a9e9f8e266a Mon Sep 17 00:00:00 2001 From: Stephen Lewis Date: Thu, 25 Jun 2026 14:03:29 -0700 Subject: [PATCH 3/4] Respect generateCode for product-specific generation --- mmv1/provider/terraform.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mmv1/provider/terraform.go b/mmv1/provider/terraform.go index fc8a72daaa8f..01877b911d82 100644 --- a/mmv1/provider/terraform.go +++ b/mmv1/provider/terraform.go @@ -462,6 +462,9 @@ func (t Terraform) getCommonCopyFiles(versionName string, generateCode, generate // Case 0: If we're generating a specific product, only copy files for that product. if t.Product != nil { + if !generateCode { + return commonCopyFiles + } googleDir := "google" if versionName != "ga" { googleDir = fmt.Sprintf("google-%s", versionName) From 44ffd124e7171efd4c67db04cb2462f3f0ab7f31 Mon Sep 17 00:00:00 2001 From: Stephen Lewis Date: Thu, 25 Jun 2026 14:35:44 -0700 Subject: [PATCH 4/4] Added fwprovider version guards for firebase --- .../fwprovider/framework_provider_access_token_test.go.tmpl | 2 ++ .../terraform/fwprovider/framework_provider_scopes_test.go.tmpl | 2 ++ .../framework_provider_user_project_override_test.go.tmpl | 2 +- 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/mmv1/third_party/terraform/fwprovider/framework_provider_access_token_test.go.tmpl b/mmv1/third_party/terraform/fwprovider/framework_provider_access_token_test.go.tmpl index 0764832bd172..439f31d10a4c 100644 --- a/mmv1/third_party/terraform/fwprovider/framework_provider_access_token_test.go.tmpl +++ b/mmv1/third_party/terraform/fwprovider/framework_provider_access_token_test.go.tmpl @@ -10,7 +10,9 @@ import ( "github.com/hashicorp/terraform-plugin-testing/helper/resource" "github.com/hashicorp/terraform-provider-google/google/acctest" +{{- if ne $.TargetVersionName "ga" }} _ "github.com/hashicorp/terraform-provider-google/google/services/firebase" +{{- end }} "github.com/hashicorp/terraform-provider-google/google/envvar" ) diff --git a/mmv1/third_party/terraform/fwprovider/framework_provider_scopes_test.go.tmpl b/mmv1/third_party/terraform/fwprovider/framework_provider_scopes_test.go.tmpl index 926152e2f424..f72fb7af673a 100644 --- a/mmv1/third_party/terraform/fwprovider/framework_provider_scopes_test.go.tmpl +++ b/mmv1/third_party/terraform/fwprovider/framework_provider_scopes_test.go.tmpl @@ -9,7 +9,9 @@ import ( "github.com/hashicorp/terraform-plugin-testing/helper/resource" "github.com/hashicorp/terraform-provider-google/google/acctest" +{{- if ne $.TargetVersionName "ga" }} _ "github.com/hashicorp/terraform-provider-google/google/services/firebase" +{{- end }} "github.com/hashicorp/terraform-provider-google/google/transport" ) diff --git a/mmv1/third_party/terraform/fwprovider/framework_provider_user_project_override_test.go.tmpl b/mmv1/third_party/terraform/fwprovider/framework_provider_user_project_override_test.go.tmpl index b4fd37a01f03..cd75fd19b869 100644 --- a/mmv1/third_party/terraform/fwprovider/framework_provider_user_project_override_test.go.tmpl +++ b/mmv1/third_party/terraform/fwprovider/framework_provider_user_project_override_test.go.tmpl @@ -9,8 +9,8 @@ import ( "github.com/hashicorp/terraform-plugin-testing/helper/resource" "github.com/hashicorp/terraform-provider-google/google/acctest" - _ "github.com/hashicorp/terraform-provider-google/google/services/firebase" {{ if ne $.TargetVersionName `ga` -}} + _ "github.com/hashicorp/terraform-provider-google/google/services/firebase" "github.com/hashicorp/terraform-provider-google/google/envvar" ptu "github.com/hashicorp/terraform-provider-google/google/provider/testutils" transport_tpg "github.com/hashicorp/terraform-provider-google/google/transport"