Support multiple products and only copy/compile files for those products

mmv1/main.go

@@ -103,8 +103,9 @@ func GenerateProducts(product, resource, providerName, version, outputPath, base
 			productsToGenerate = append(productsToGenerate, p.PackagePath)
 		}
 	} else {
-		var productToGenerate = fmt.Sprintf("products/%s", product)
-		productsToGenerate = []string{productToGenerate}
+		for _, prod := range strings.Split(product, ",") {
+			productsToGenerate = append(productsToGenerate, fmt.Sprintf("products/%s", strings.TrimSpace(prod)))
+		}
 	}
 
 	for _, productApi := range loadedProducts {
@@ -122,8 +123,8 @@ func GenerateProducts(product, resource, providerName, version, outputPath, base
 	})
 
 	// In order to only copy/compile files once per provider this must be called outside
-	// of the products loop. Create an MMv1 provider with an arbitrary product (the first loaded).
-	providerToGenerate := newProvider(providerName, version, productsForVersion[0], startTime, wrappedFS)
+	// of the products loop. Create an MMv1 provider with a nil product to trigger shared file behavior.
+	providerToGenerate := newProvider(providerName, version, nil, startTime, wrappedFS)
 	providerToGenerate.CopyCommonFiles(outputPath, generateCode, generateDocs)
 
 	if generateCode {
@@ -148,6 +149,11 @@ func GenerateProduct(version, providerName string, productApi *api.Product, outp
 	log.Printf("%s: Generating files", productApi.PackagePath)
 	providerToGenerate := newProvider(providerName, version, productApi, startTime, fsys)
 	providerToGenerate.Generate(outputPath, resourceToGenerate, generateCode, generateDocs)
+
+	providerToGenerate.CopyCommonFiles(outputPath, generateCode, generateDocs)
+	if generateCode {
+		providerToGenerate.CompileCommonFiles(outputPath, []*api.Product{productApi}, "")
+	}
 }
 
 func newProvider(providerName, version string, productApi *api.Product, startTime time.Time, fsys fs.FS) provider.Provider {

mmv1/provider/terraform.go

@@ -59,14 +59,16 @@ func NewTerraform(product *api.Product, versionName string, startTime time.Time,
 		IAMResourceCount:  0,
 		Product:           product,
 		TargetVersionName: versionName,
-		Version:           *product.VersionObjOrClosest(versionName),
 		StartTime:         startTime,
 		templateFS:        templateFS,
 	}
 
-	t.Product.ImportPath = ImportPathFromVersion(versionName)
-	for _, r := range t.Product.Objects {
-		r.ImportPath = t.Product.ImportPath
+	if product != nil {
+		t.Version = *product.VersionObjOrClosest(versionName)
+		t.Product.ImportPath = ImportPathFromVersion(versionName)
+		for _, r := range t.Product.Objects {
+			r.ImportPath = t.Product.ImportPath
+		}
 	}
 
 	return t
@@ -458,6 +460,20 @@ func (t Terraform) getCommonCopyFiles(versionName string, generateCode, generate
 	// key is the target file and value is the source file
 	commonCopyFiles := make(map[string]string, 0)
 
+	// Case 0: If we're generating a specific product, only copy files for that product.
+	if t.Product != nil {
+		if !generateCode {
+			return commonCopyFiles
+		}
+		googleDir := "google"
+		if versionName != "ga" {
+			googleDir = fmt.Sprintf("google-%s", versionName)
+		}
+		files := t.getCopyFilesInFolder("third_party/terraform/services/"+t.Product.ApiName, googleDir)
+		maps.Copy(commonCopyFiles, files)
+		return commonCopyFiles
+	}
+
 	// Case 1: When copy all of files except .tmpl in a folder to the root directory of downstream repository,
 	// save the folder name to foldersCopiedToRootDir
 	foldersCopiedToRootDir := []string{"third_party/terraform/META.d", "third_party/terraform/version"}
@@ -493,7 +509,6 @@ func (t Terraform) getCommonCopyFiles(versionName string, generateCode, generate
 			"third_party/terraform/fwvalidators",
 			"third_party/terraform/provider",
 			"third_party/terraform/registry",
-			"third_party/terraform/services",
 			"third_party/terraform/sweeper",
 			"third_party/terraform/test-fixtures",
 			"third_party/terraform/tpgdclresource",
@@ -596,7 +611,9 @@ func (t Terraform) CopyFileList(outputFolder string, files map[string]string, ge
 // Compiles files that are shared at the provider level
 func (t Terraform) CompileCommonFiles(outputFolder string, products []*api.Product, overridePath string) {
 	log.Printf("Generating common files for %s", ProviderName(t))
-	t.generateResourcesForVersion(products)
+	if t.Product == nil {
+		t.generateResourcesForVersion(products)
+	}
 	files := t.getCommonCompileFiles(t.TargetVersionName)
 	templateData := NewTemplateData(outputFolder, t.TargetVersionName, t.templateFS)
 	t.CompileFileList(outputFolder, files, *templateData, products)
@@ -608,6 +625,14 @@ func (t Terraform) getCommonCompileFiles(versionName string) map[string]string {
 	// key is the target file and the value is the source file
 	commonCompileFiles := make(map[string]string, 0)
 
+	if t.Product != nil {
+		googleDir := "google"
+		if versionName != "ga" {
+			googleDir = fmt.Sprintf("google-%s", versionName)
+		}
+		return t.getCompileFilesInFolder("third_party/terraform/services/"+t.Product.ApiName, googleDir)
+	}
+
 	// Case 1: When compile all of files except .tmpl in a folder to the root directory of downstream repository,
 	// save the folder name to foldersCopiedToRootDir
 	foldersCompiledToRootDir := []string{"third_party/terraform/scripts"}
@@ -628,7 +653,6 @@ func (t Terraform) getCommonCompileFiles(versionName string) map[string]string {
 		"third_party/terraform/fwresource",
 		"third_party/terraform/fwtransport",
 		"third_party/terraform/provider",
-		"third_party/terraform/services",
 		"third_party/terraform/sweeper",
 		"third_party/terraform/test-fixtures",
 		"third_party/terraform/tpgdclresource",

mmv1/provider/terraform_tgc.go

@@ -58,8 +58,10 @@ func NewTerraformGoogleConversion(product *api.Product, versionName string, star
 		templateFS:        templateFS,
 	}
 
-	for _, r := range t.Product.Objects {
-		r.ImportPath = ImportPathFromVersion(versionName)
+	if product != nil {
+		for _, r := range t.Product.Objects {
+			r.ImportPath = ImportPathFromVersion(versionName)
+		}
 	}
 
 	return t
@@ -185,23 +187,7 @@ func (tgc *TerraformGoogleConversion) generateCaiIamResources(products []*api.Pr
 func (tgc TerraformGoogleConversion) CompileCommonFiles(outputFolder string, products []*api.Product, overridePath string) {
 	log.Printf("Compiling common files for tgc.")
 
-	tgc.generateCaiIamResources(products)
-	tgc.NonDefinedTests = retrieveFullManifestOfNonDefinedTests(tgc.templateFS)
-
-	files := retrieveFullListOfTestFiles(tgc.templateFS)
-	for _, file := range files {
-		tgc.Tests = append(tgc.Tests, strings.Split(file, ".")[0])
-	}
-	tgc.Tests = slices.Compact(tgc.Tests)
-
-	testSource := make(map[string]string)
-	for target, source := range retrieveTestSourceCodeWithLocation(tgc.templateFS, ".tmpl") {
-		target := strings.Replace(target, "go.tmpl", "go", 1)
-		testSource[target] = source
-	}
-
 	templateData := NewTemplateData(outputFolder, tgc.TargetVersionName, tgc.templateFS)
-	tgc.CompileFileList(outputFolder, testSource, *templateData, products)
 
 	resourceConverters := map[string]string{
 		"converters/google/resources/resource_converters.go":                       "third_party/tgc/resource_converters.go.tmpl",
@@ -220,7 +206,40 @@ func (tgc TerraformGoogleConversion) CompileCommonFiles(outputFolder string, pro
 		"converters/google/resources/services/spanner/iam_spanner_database.go":     "third_party/terraform/services/spanner/iam_spanner_database.go.tmpl",
 		"converters/google/resources/services/spanner/iam_spanner_instance.go":     "third_party/terraform/services/spanner/iam_spanner_instance.go.tmpl",
 	}
-	tgc.CompileFileList(outputFolder, resourceConverters, *templateData, products)
+
+	filteredFiles := make(map[string]string)
+	if tgc.Product != nil {
+		for target, source := range resourceConverters {
+			if strings.Contains(target, "/services/"+tgc.Product.ApiName+"/") {
+				filteredFiles[target] = source
+			}
+		}
+		tgc.CompileFileList(outputFolder, filteredFiles, *templateData, products)
+	} else {
+		// Shared compilation
+		tgc.generateCaiIamResources(products)
+		tgc.NonDefinedTests = retrieveFullManifestOfNonDefinedTests(tgc.templateFS)
+
+		files := retrieveFullListOfTestFiles(tgc.templateFS)
+		for _, file := range files {
+			tgc.Tests = append(tgc.Tests, strings.Split(file, ".")[0])
+		}
+		tgc.Tests = slices.Compact(tgc.Tests)
+
+		testSource := make(map[string]string)
+		for target, source := range retrieveTestSourceCodeWithLocation(tgc.templateFS, ".tmpl") {
+			target := strings.Replace(target, "go.tmpl", "go", 1)
+			testSource[target] = source
+		}
+		tgc.CompileFileList(outputFolder, testSource, *templateData, products)
+
+		for target, source := range resourceConverters {
+			if !strings.Contains(target, "/services/") {
+				filteredFiles[target] = source
+			}
+		}
+		tgc.CompileFileList(outputFolder, filteredFiles, *templateData, products)
+	}
 }
 
 func (tgc TerraformGoogleConversion) CompileFileList(outputFolder string, files map[string]string, fileTemplate TemplateData, products []*api.Product) {
@@ -359,9 +378,6 @@ func (tgc TerraformGoogleConversion) CopyCommonFiles(outputFolder string, genera
 		return
 	}
 
-	tgc.CopyFileList(outputFolder, retrieveFullListOfTestTilesWithLocation(tgc.templateFS))
-	tgc.CopyFileList(outputFolder, retrieveTestSourceCodeWithLocation(tgc.templateFS, ".go"))
-
 	resourceConverters := map[string]string{
 		"../caiasset/asset.go":                                                                  "third_party/tgc/caiasset/asset.go",
 		"converters/google/resources/cai/constants.go":                                          "third_party/tgc/cai/constants.go",
@@ -456,7 +472,25 @@ func (tgc TerraformGoogleConversion) CopyCommonFiles(outputFolder string, genera
 		"tfplan2cai.go":                                                                         "third_party/tgc/tfplan2cai.go",
 		"tfplan_to_cai_test.go":                                                                 "third_party/tgc/tfplan_to_cai_test.go",
 	}
-	tgc.CopyFileList(outputFolder, resourceConverters)
+
+	filteredFiles := make(map[string]string)
+	if tgc.Product != nil {
+		for target, source := range resourceConverters {
+			if strings.Contains(target, "/services/"+tgc.Product.ApiName+"/") {
+				filteredFiles[target] = source
+			}
+		}
+	} else {
+		// Shared files
+		tgc.CopyFileList(outputFolder, retrieveFullListOfTestTilesWithLocation(tgc.templateFS))
+		tgc.CopyFileList(outputFolder, retrieveTestSourceCodeWithLocation(tgc.templateFS, ".go"))
+		for target, source := range resourceConverters {
+			if !strings.Contains(target, "/services/") {
+				filteredFiles[target] = source
+			}
+		}
+	}
+	tgc.CopyFileList(outputFolder, filteredFiles)
 }
 
 func (tgc TerraformGoogleConversion) CopyFileList(outputFolder string, files map[string]string) {

mmv1/provider/terraform_tgc_cai2hcl.go

@@ -14,10 +14,13 @@
 package provider
 
 import (
+	"errors"
 	"fmt"
 	"io/fs"
 	"log"
 	"os"
+	"path/filepath"
+	"strings"
 	"time"
 
 	"github.com/GoogleCloudPlatform/magic-modules/mmv1/api"
@@ -43,8 +46,10 @@ func NewCaiToTerraformConversion(product *api.Product, versionName string, start
 		templateFS:        templateFS,
 	}
 
-	for _, r := range t.Product.Objects {
-		r.ImportPath = ImportPathFromVersion(versionName)
+	if product != nil {
+		for _, r := range t.Product.Objects {
+			r.ImportPath = ImportPathFromVersion(versionName)
+		}
 	}
 
 	return t
@@ -66,7 +71,19 @@ func (cai2hcl CaiToTerraformConversion) CopyCommonFiles(outputFolder string, gen
 		log.Println(fmt.Errorf("error creating output directory %v: %v", outputFolder, err))
 	}
 
-	if err := copy.Copy("third_party/cai2hcl", outputFolder); err != nil {
-		log.Println(fmt.Errorf("error copying directory %v: %v", outputFolder, err))
+	if cai2hcl.Product != nil {
+		srcDir := filepath.Join("third_party/cai2hcl/services", cai2hcl.Product.ApiName)
+		dstDir := filepath.Join(outputFolder, "services", cai2hcl.Product.ApiName)
+		if err := copy.Copy(srcDir, dstDir); err != nil && !errors.Is(err, os.ErrNotExist) {
+			log.Println(fmt.Errorf("error copying service directory %v: %v", srcDir, err))
+		}
+	} else {
+		if err := copy.Copy("third_party/cai2hcl", outputFolder, copy.Options{
+			Skip: func(srcinfo os.FileInfo, src, dest string) (bool, error) {
+				return strings.Contains(src, "/services/"), nil
+			},
+		}); err != nil {
+			log.Println(fmt.Errorf("error copying directory %v: %v", outputFolder, err))
+		}
 	}
 }

mmv1/provider/terraform_tgc_next.go

@@ -75,9 +75,11 @@ func NewTerraformGoogleConversionNext(product *api.Product, versionName string,
 		templateFS:                 templateFS,
 	}
 
-	t.Product.ImportPath = ImportPathFromVersion(versionName)
-	for _, r := range t.Product.Objects {
-		r.ImportPath = ImportPathFromVersion(versionName)
+	if product != nil {
+		t.Product.ImportPath = ImportPathFromVersion(versionName)
+		for _, r := range t.Product.Objects {
+			r.ImportPath = ImportPathFromVersion(versionName)
+		}
 	}
 
 	return t
@@ -180,8 +182,6 @@ func (tgc *TerraformGoogleConversionNext) GenerateProduct(outputFolder string) {
 }
 
 func (tgc TerraformGoogleConversionNext) CompileCommonFiles(outputFolder string, products []*api.Product, overridePath string) {
-	tgc.generateResourcesForVersion(products)
-
 	resourceConverters := map[string]string{
 		// common
 		"pkg/transport/config.go":                    "third_party/terraform/transport/config.go.tmpl",
@@ -206,7 +206,25 @@ func (tgc TerraformGoogleConversionNext) CompileCommonFiles(outputFolder string,
 	}
 
 	templateData := NewTemplateData(outputFolder, tgc.TargetVersionName, tgc.templateFS)
-	tgc.CompileFileList(outputFolder, resourceConverters, *templateData, products)
+	filteredFiles := make(map[string]string)
+
+	if tgc.Product != nil {
+		for target, source := range resourceConverters {
+			if strings.Contains(target, "/services/"+tgc.Product.ApiName+"/") {
+				filteredFiles[target] = source
+			}
+		}
+		tgc.CompileFileList(outputFolder, filteredFiles, *templateData, products)
+	} else {
+		// Shared compilation
+		tgc.generateResourcesForVersion(products)
+		for target, source := range resourceConverters {
+			if !strings.Contains(target, "/services/") {
+				filteredFiles[target] = source
+			}
+		}
+		tgc.CompileFileList(outputFolder, filteredFiles, *templateData, products)
+	}
 }
 
 func (tgc TerraformGoogleConversionNext) CompileFileList(outputFolder string, files map[string]string, fileTemplate TemplateData, products []*api.Product) {
@@ -249,10 +267,6 @@ func (tgc TerraformGoogleConversionNext) CopyCommonFiles(outputFolder string, ge
 		log.Println(fmt.Errorf("error creating output directory %v: %v", outputFolder, err))
 	}
 
-	if err := copy.Copy("third_party/tgc_next", outputFolder); err != nil {
-		log.Println(fmt.Errorf("error copying directory %v: %v", outputFolder, err))
-	}
-
 	resourceConverters := map[string]string{
 		// common
 		"pkg/envvar/envvar_utils.go":              "third_party/terraform/envvar/envvar_utils.go",
@@ -284,7 +298,40 @@ func (tgc TerraformGoogleConversionNext) CopyCommonFiles(outputFolder string, ge
 		"pkg/services/resourcemanagerv3/client.go":      "third_party/terraform/services/resourcemanagerv3/client.go",
 		"pkg/services/storage/client.go":                "third_party/terraform/services/storage/client.go",
 	}
-	tgc.CopyFileList(outputFolder, resourceConverters)
+
+	filteredFiles := make(map[string]string)
+
+	if tgc.Product != nil {
+		// Service-specific copying
+		for target, source := range resourceConverters {
+			if strings.Contains(target, "/services/"+tgc.Product.ApiName+"/") {
+				filteredFiles[target] = source
+			}
+		}
+		tgc.CopyFileList(outputFolder, filteredFiles)
+
+		srcDir := filepath.Join("third_party/tgc_next/pkg/services", tgc.Product.ApiName)
+		dstDir := filepath.Join(outputFolder, "pkg/services", tgc.Product.ApiName)
+		if err := copy.Copy(srcDir, dstDir); err != nil && !errors.Is(err, os.ErrNotExist) {
+			log.Println(fmt.Errorf("error copying service directory %v: %v", srcDir, err))
+		}
+	} else {
+		// Shared copying
+		if err := copy.Copy("third_party/tgc_next", outputFolder, copy.Options{
+			Skip: func(srcinfo os.FileInfo, src, dest string) (bool, error) {
+				return strings.Contains(src, "pkg/services/"), nil
+			},
+		}); err != nil {
+			log.Println(fmt.Errorf("error copying directory %v: %v", outputFolder, err))
+		}
+
+		for target, source := range resourceConverters {
+			if !strings.Contains(target, "/services/") {
+				filteredFiles[target] = source
+			}
+		}
+		tgc.CopyFileList(outputFolder, filteredFiles)
+	}
 }
 
 func (tgc TerraformGoogleConversionNext) CopyTfToCaiCommonFiles(outputFolder string) {

mmv1/third_party/terraform/fwprovider/framework_provider_access_token_test.go.tmpl

@@ -10,7 +10,9 @@ import (
 
 	"github.com/hashicorp/terraform-plugin-testing/helper/resource"
 	"github.com/hashicorp/terraform-provider-google/google/acctest"
+{{- if ne $.TargetVersionName "ga" }}
 	_ "github.com/hashicorp/terraform-provider-google/google/services/firebase"
+{{- end }}
 	"github.com/hashicorp/terraform-provider-google/google/envvar"
 )