feat(filter): add exclude_services support across providers and CLI (#749)

internal/runner/options.go

@@ -32,6 +32,7 @@ type Options struct {
 	Providers          goflags.StringSlice // Providers specifies what providers to fetch assets for.
 	Id                 goflags.StringSlice // Id specifies what id's to fetch assets for.
 	Services           goflags.StringSlice // Services specifies what services to fetch assets for a provider.
+	ExcludeServices    goflags.StringSlice // ExcludeServices specifies what services to exclude for a provider.
 	ExtendedMetadata   bool                // ExtendedMetadata enables extended metadata for providers.
 	ProviderConfig     string              // ProviderConfig is the location of the provider config file.
 	DisableUpdateCheck bool                // DisableUpdateCheck disable automatic update check
@@ -85,6 +86,7 @@ func ParseOptions() *Options {
 		flagSet.BoolVar(&options.IPAddress, "ip", false, "display only ips in results"),
 		flagSet.BoolVar(&options.ExtendedMetadata, "extended-metadata", false, "enable extended metadata for providers"),
 		flagSet.StringSliceVarP(&options.Services, "service", "s", nil, "query and display results from given service (comma-separated)) (default "+strings.Join(defaultServies, ",")+")", goflags.CommaSeparatedStringSliceOptions),
+		flagSet.StringSliceVarP(&options.ExcludeServices, "exclude-service", "es", nil, "exclude given services from query results (comma-separated)", goflags.CommaSeparatedStringSliceOptions),
 		flagSet.BoolVarP(&options.ExcludePrivate, "exclude-private", "ep", false, "exclude private ips in cli output"),
 	)
 	flagSet.CreateGroup("update", "Update",

internal/runner/runner.go

@@ -38,6 +38,9 @@ func New(options *Options) (*Runner, error) {
 	if len(options.Services) == 0 {
 		options.Services = append(options.Services, config.GetServiceNames()...)
 	}
+	if len(options.ExcludeServices) == 0 {
+		options.ExcludeServices = append(options.ExcludeServices, config.GetExcludeServiceNames()...)
+	}
 
 	// assign default services if not provided
 	if len(options.Services) == 0 {
@@ -57,6 +60,10 @@ func (r *Runner) Enumerate() {
 	if r.options.Services != nil {
 		services = r.options.Services
 	}
+	excludeServices := []string{}
+	if r.options.ExcludeServices != nil {
+		excludeServices = r.options.ExcludeServices
+	}
 
 	for _, item := range r.config {
 		if item == nil {
@@ -68,6 +75,9 @@ func (r *Runner) Enumerate() {
 		if len(services) > 0 {
 			item["services"] = strings.Join(services, ",")
 		}
+		if len(excludeServices) > 0 {
+			item["exclude_services"] = strings.Join(excludeServices, ",")
+		}
 		if r.options.ExtendedMetadata {
 			item["extended_metadata"] = "true"
 		}

pkg/providers/alibaba/alibaba.go

@@ -59,6 +59,11 @@ func New(options schema.OptionBlock) (*Provider, error) {
 			services[s] = struct{}{}
 		}
 	}
+	if es, ok := options.GetMetadata("exclude_services"); ok {
+		for _, s := range strings.Split(es, ",") {
+			delete(services, strings.TrimSpace(s))
+		}
+	}
 	provider.services = services
 
 	if services.Has("instance") {

pkg/providers/arvancloud/arvancloud.go

@@ -53,6 +53,11 @@ func New(options schema.OptionBlock) (*Provider, error) {
 			services[s] = struct{}{}
 		}
 	}
+	if es, ok := options.GetMetadata("exclude_services"); ok {
+		for _, s := range strings.Split(es, ",") {
+			delete(services, strings.TrimSpace(s))
+		}
+	}
 
 	return &Provider{id: id, client: api, services: services}, nil
 }

pkg/providers/aws/aws.go

@@ -101,6 +101,11 @@ func (p *ProviderOptions) ParseOptionBlock(block schema.OptionBlock) error {
 			services[s] = struct{}{}
 		}
 	}
+	if es, ok := block.GetMetadata("exclude_services"); ok {
+		for _, s := range strings.Split(es, ",") {
+			delete(services, strings.TrimSpace(s))
+		}
+	}
 	p.Services = services
 
 	if extendedMetadata, ok := block.GetMetadata("extended_metadata"); ok {

pkg/providers/azure/azure.go

@@ -61,6 +61,11 @@ func New(options schema.OptionBlock) (*Provider, error) {
 			services[s] = struct{}{}
 		}
 	}
+	if es, ok := options.GetMetadata("exclude_services"); ok {
+		for _, s := range strings.Split(es, ",") {
+			delete(services, strings.TrimSpace(s))
+		}
+	}
 
 	provider := &Provider{
 		Credential: credential, // Track 2: use credential instead of authorizer

pkg/providers/cloudflare/cloudflare.go

@@ -42,6 +42,11 @@ func New(options schema.OptionBlock) (*Provider, error) {
 			services[s] = struct{}{}
 		}
 	}
+	if es, ok := options.GetMetadata("exclude_services"); ok {
+		for _, s := range strings.Split(es, ",") {
+			delete(services, strings.TrimSpace(s))
+		}
+	}
 
 	// Parse extended metadata option
 	extendedMetadata := false

pkg/providers/custom/custom.go

@@ -102,6 +102,11 @@ func (p *ProviderOptions) ParseOptionBlock(block schema.OptionBlock) error {
 			services[s] = struct{}{}
 		}
 	}
+	if es, ok := block.GetMetadata("exclude_services"); ok {
+		for _, s := range strings.Split(es, ",") {
+			delete(services, strings.TrimSpace(s))
+		}
+	}
 
 	np, err := networkpolicy.New(networkpolicy.DefaultOptions)
 	if err != nil {

pkg/providers/digitalocean/digitalocean.go

@@ -43,6 +43,11 @@ func New(options schema.OptionBlock) (*Provider, error) {
 			services[s] = struct{}{}
 		}
 	}
+	if es, ok := options.GetMetadata("exclude_services"); ok {
+		for _, s := range strings.Split(es, ",") {
+			delete(services, strings.TrimSpace(s))
+		}
+	}
 
 	// Check for extended metadata option
 	extendedMetadata := false

pkg/providers/dnssimple/dnssimple.go

@@ -70,6 +70,11 @@ func New(options schema.OptionBlock) (*Provider, error) {
 			services[s] = struct{}{}
 		}
 	}
+	if es, ok := options.GetMetadata("exclude_services"); ok {
+		for _, s := range strings.Split(es, ",") {
+			delete(services, strings.TrimSpace(s))
+		}
+	}
 
 	provider := &Provider{
 		id:       id,

pkg/providers/gcp/gcp.go

@@ -256,6 +256,11 @@ func newIndividualProvider(options schema.OptionBlock, id, JSONData string) (*Pr
 			services[s] = struct{}{}
 		}
 	}
+	if es, ok := options.GetMetadata("exclude_services"); ok {
+		for _, s := range strings.Split(es, ",") {
+			delete(services, strings.TrimSpace(s))
+		}
+	}
 	provider.services = services
 
 	configuredProjects := getProjectIDsFromOptions(options)

pkg/providers/heroku/heroku.go

@@ -50,6 +50,11 @@ func New(options schema.OptionBlock) (*Provider, error) {
 			services[s] = struct{}{}
 		}
 	}
+	if es, ok := options.GetMetadata("exclude_services"); ok {
+		for _, s := range strings.Split(es, ",") {
+			delete(services, strings.TrimSpace(s))
+		}
+	}
 
 	return &Provider{id: id, client: heroku.NewService(heroku.DefaultClient), services: services}, nil
 }

pkg/providers/hetzner/hetzner.go

@@ -49,6 +49,11 @@ func New(options schema.OptionBlock) (*Provider, error) {
 			services[s] = struct{}{}
 		}
 	}
+	if es, ok := options.GetMetadata("exclude_services"); ok {
+		for _, s := range strings.Split(es, ",") {
+			delete(services, strings.TrimSpace(s))
+		}
+	}
 	return &Provider{id: id, client: hetzner.NewClient(opts), services: services}, nil
 }
 

pkg/providers/k8s/kubernetes.go

@@ -81,6 +81,11 @@ func New(options schema.OptionBlock) (*Provider, error) {
 			services[s] = struct{}{}
 		}
 	}
+	if es, ok := options.GetMetadata("exclude_services"); ok {
+		for _, s := range strings.Split(es, ",") {
+			delete(services, strings.TrimSpace(s))
+		}
+	}
 	var providerExtendedMetadata bool
 	if extendedMetadata, ok := options.GetMetadata("extended_metadata"); ok {
 		providerExtendedMetadata = extendedMetadata == "true"

pkg/providers/linode/linode.go

@@ -59,6 +59,11 @@ func New(options schema.OptionBlock) (*Provider, error) {
 			services[s] = struct{}{}
 		}
 	}
+	if es, ok := options.GetMetadata("exclude_services"); ok {
+		for _, s := range strings.Split(es, ",") {
+			delete(services, strings.TrimSpace(s))
+		}
+	}
 	return &Provider{id: id, client: &client, services: services}, nil
 }
 

pkg/providers/namecheap/namecheap.go

@@ -69,6 +69,11 @@ func New(options schema.OptionBlock) (*Provider, error) {
 			services[s] = struct{}{}
 		}
 	}
+	if es, ok := options.GetMetadata("exclude_services"); ok {
+		for _, s := range strings.Split(es, ",") {
+			delete(services, strings.TrimSpace(s))
+		}
+	}
 
 	return &Provider{id: id, client: namecheap.NewClient(&clientOptions), services: services}, nil
 }

pkg/providers/openstack/openstack.go

@@ -99,6 +99,11 @@ func New(options schema.OptionBlock) (*Provider, error) {
 			services[s] = struct{}{}
 		}
 	}
+	if es, ok := options.GetMetadata("exclude_services"); ok {
+		for _, s := range strings.Split(es, ",") {
+			delete(services, strings.TrimSpace(s))
+		}
+	}
 	return &Provider{id: id, client: client, services: services}, nil
 }
 

pkg/providers/ovh/ovh.go

@@ -39,6 +39,11 @@ func New(options schema.OptionBlock) (*Provider, error) {
 			services[s] = struct{}{}
 		}
 	}
+	if es, ok := options.GetMetadata("exclude_services"); ok {
+		for _, s := range strings.Split(es, ",") {
+			delete(services, strings.TrimSpace(s))
+		}
+	}
 
 	// OVH endpoint (default ovh-eu)
 	endpoint := "ovh-eu"

pkg/providers/scaleway/scaleway.go

@@ -46,6 +46,11 @@ func New(options schema.OptionBlock) (*Provider, error) {
 			services[s] = struct{}{}
 		}
 	}
+	if es, ok := options.GetMetadata("exclude_services"); ok {
+		for _, s := range strings.Split(es, ",") {
+			delete(services, strings.TrimSpace(s))
+		}
+	}
 
 	client, err := scw.NewClient(scw.WithAuth(accessKey, accessToken))
 	if err != nil {

pkg/providers/terraform/terraform.go

@@ -46,6 +46,11 @@ func New(options schema.OptionBlock) (*Provider, error) {
 			services[s] = struct{}{}
 		}
 	}
+	if es, ok := options.GetMetadata("exclude_services"); ok {
+		for _, s := range strings.Split(es, ",") {
+			delete(services, strings.TrimSpace(s))
+		}
+	}
 	return &Provider{path: StatePathFile, id: id, services: services}, nil
 }
 

pkg/providers/vercel/vercel.go

@@ -42,6 +42,11 @@ func New(options schema.OptionBlock) (*Provider, error) {
 			services[s] = struct{}{}
 		}
 	}
+	if es, ok := options.GetMetadata("exclude_services"); ok {
+		for _, s := range strings.Split(es, ",") {
+			delete(services, strings.TrimSpace(s))
+		}
+	}
 
 	client := newAPIClient(newClientConfig{
 		Token:  accessKey,

pkg/schema/schema.go

@@ -226,6 +226,22 @@ func (o Options) GetServiceNames() []string {
 	return services
 }
 
+// GetExcludeServiceNames returns the excluded services from the options
+func (o Options) GetExcludeServiceNames() []string {
+	services := make([]string, 0)
+	for _, option := range o {
+		if serviceNameList, ok := option["exclude_services"]; ok {
+			for _, serviceName := range strings.Split(serviceNameList, ",") {
+				trimmedServiceName := strings.TrimSpace(serviceName)
+				if trimmedServiceName != "" {
+					services = append(services, trimmedServiceName)
+				}
+			}
+		}
+	}
+	return services
+}
+
 // OptionBlock is a single option on which operation is possible
 type OptionBlock map[string]string
 
@@ -240,7 +256,7 @@ func (ob *OptionBlock) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	// Convert raw map to OptionBlock and handle special cases
 	for key, value := range rawMap {
 		switch key {
-		case "account_ids", "exclude_account_ids", "urls", "services", "project_ids", "exclude_project_ids":
+		case "account_ids", "exclude_account_ids", "urls", "services", "exclude_services", "project_ids", "exclude_project_ids":
 			if valueArr, ok := value.([]interface{}); ok {
 				var strArr []string
 				for _, v := range valueArr {
@@ -296,6 +312,41 @@ func (o OptionBlock) GetMetadata(key string) (string, bool) {
 	return data, true
 }
 
+// ParseServices parses services and exclude_services metadata from option block against supported services
+func (o OptionBlock) ParseServices(supportedServices []string) ServiceMap {
+	supportedServicesMap := make(map[string]struct{})
+	for _, s := range supportedServices {
+		supportedServicesMap[s] = struct{}{}
+	}
+
+	services := make(ServiceMap)
+	if ss, ok := o.GetMetadata("services"); ok {
+		for _, s := range strings.Split(ss, ",") {
+			s = strings.TrimSpace(s)
+			if _, ok := supportedServicesMap[s]; ok {
+				services[s] = struct{}{}
+			}
+		}
+	}
+
+	// if no services explicitly specified, start with all supported services
+	if len(services) == 0 {
+		for _, s := range supportedServices {
+			services[s] = struct{}{}
+		}
+	}
+
+	// subtract exclude_services if specified
+	if es, ok := o.GetMetadata("exclude_services"); ok {
+		for _, s := range strings.Split(es, ",") {
+			s = strings.TrimSpace(s)
+			delete(services, s)
+		}
+	}
+
+	return services
+}
+
 type ServiceMap map[string]struct{}
 
 func (s ServiceMap) Has(service string) bool {

pkg/schema/schema_test.go

@@ -89,3 +89,29 @@ func TestOptionBlockScalarFallback(t *testing.T) {
 	require.True(t, ok)
 	require.Equal(t, "PDScannerRole", value)
 }
+
+func TestOptionBlockParsesExcludeServices(t *testing.T) {
+	data := `
+- provider: gcp
+  exclude_services:
+    - cloud-function
+    - cloud-run
+`
+	var options Options
+	err := yaml.Unmarshal([]byte(data), &options)
+	require.NoError(t, err)
+	require.Len(t, options, 1)
+
+	value, ok := options[0].GetMetadata("exclude_services")
+	require.True(t, ok)
+	require.Equal(t, "cloud-function,cloud-run", value)
+	require.Equal(t, []string{"cloud-function", "cloud-run"}, options.GetExcludeServiceNames())
+
+	supported := []string{"dns", "compute", "gke", "cloud-function", "cloud-run"}
+	serviceMap := options[0].ParseServices(supported)
+	require.True(t, serviceMap.Has("dns"))
+	require.True(t, serviceMap.Has("compute"))
+	require.True(t, serviceMap.Has("gke"))
+	require.False(t, serviceMap.Has("cloud-function"))
+	require.False(t, serviceMap.Has("cloud-run"))
+}