Merge pull request #746 from projectdiscovery/feat/gcp-exclude-project-ids

feat(gcp): exclude project ids

Author: ShubhamRasal

PROVIDERS.md

@@ -153,6 +153,7 @@ Google Cloud Platform supports **two discovery approaches** and **two authentica
 - `source_credentials` (string, optional): Path to source credentials file (uses ADC if not provided)
 - `token_lifetime` (string, optional): Token lifetime in seconds (e.g., "3600s") or Go duration format (e.g., "1h"). Range: 1s to 3600s (1 hour). Default: "3600s"
 - `project_ids` (list, optional): Comma-separated/list of project IDs to enumerate. When provided, Cloudlist skips discovery in every other accessible project, both for individual APIs and the organization-level Asset API.
+- `exclude_project_ids` (list, optional): Comma-separated/list of project IDs to exclude from organization-wide discovery. Requires `organization_id`. Mutually exclusive with `project_ids`. When provided, Cloudlist discovers all projects in the organization and skips the excluded ones, using per-project Asset API calls for the remaining projects.
 
 ---
 
@@ -199,6 +200,20 @@ Google Cloud Platform supports **two discovery approaches** and **two authentica
 
 Add `project_ids` to either configuration style to limit enumeration strictly to the listed projects (Cloud Asset API requests are filtered too), which is helpful for large organizations or delegated-access service accounts.
 
+**Excluding Projects:**
+
+```yaml
+- provider: gcp
+  organization_id: "123456789012"
+  gcp_service_account_key: '$GCP_SA_KEY'
+  exclude_project_ids:
+    - sandbox-project
+    - legacy-app
+    - test-environment
+```
+
+Use `exclude_project_ids` to scan all projects in an organization except the listed ones. This is useful when only a few projects need to be excluded from a large organization.
+
 **Required Organization-Level Roles:**
 1. `roles/cloudasset.viewer` - Core Asset API access
 2. `roles/resourcemanager.viewer` - List projects in organization

pkg/providers/gcp/gcp.go

@@ -205,6 +205,18 @@ func New(options schema.OptionBlock) (schema.Provider, error) {
 
 	gologger.Info().Msgf("Creating GCP provider with id: %s", id)
 
+	// Validate exclude_project_ids constraints
+	hasInclude := len(getProjectIDsFromOptions(options)) > 0
+	hasExclude := len(getExcludeProjectIDsFromOptions(options)) > 0
+	if hasInclude && hasExclude {
+		return nil, errkit.New("project_ids and exclude_project_ids are mutually exclusive")
+	}
+	if hasExclude {
+		if _, ok := options.GetMetadata("organization_id"); !ok {
+			return nil, errkit.New("exclude_project_ids requires organization_id to be set")
+		}
+	}
+
 	// Note: gcp_service_account_key is optional
 	// Authentication will fall back to Application Default Credentials (ADC) if not provided
 	// This works for both traditional and short-lived credential modes
@@ -798,6 +810,46 @@ func newOrganizationProvider(options schema.OptionBlock, id, JSONData, organizat
 		if len(projects) == 0 {
 			gologger.Info().Msgf("No projects listed, will use organization-level Asset API discovery for org %s", organizationID)
 		}
+
+		// Apply exclude filter if configured
+		excludeIDs := getExcludeProjectIDsFromOptions(options)
+		if len(excludeIDs) > 0 {
+			if len(projects) == 0 {
+				return nil, errkit.New("exclude_project_ids requires project listing to succeed, but no projects were discovered")
+			}
+			excludeScope := newProjectScope(excludeIDs)
+			if excludeScope != nil {
+				if manager != nil {
+					if err := excludeScope.enrichWithProjectNumbers(context.Background(), manager); err != nil {
+						gologger.Warning().Msgf("Could not resolve excluded project ids: %s", err)
+					}
+				}
+				filtered := make([]string, 0, len(projects))
+				for _, p := range projects {
+					if !excludeScope.containsID(p) {
+						filtered = append(filtered, p)
+					}
+				}
+				matched := len(projects) - len(filtered)
+				gologger.Info().Msgf("Excluded %d/%d project(s), %d remaining", matched, len(excludeIDs), len(filtered))
+				projects = filtered
+			}
+
+			if len(projects) == 0 {
+				return nil, errkit.New("all projects were excluded, no projects remaining for discovery")
+			}
+
+			// Build projectScope from remaining projects so Resources() uses per-project Asset API path
+			scope := newProjectScope(projects)
+			if scope != nil {
+				if manager != nil {
+					if err := scope.enrichWithProjectNumbers(context.Background(), manager); err != nil {
+						gologger.Warning().Msgf("Could not resolve remaining project ids: %s", err)
+					}
+				}
+				provider.projectScope = scope
+			}
+		}
 	}
 	provider.projects = projects
 
@@ -1034,6 +1086,14 @@ func getProjectIDsFromOptions(options schema.OptionBlock) []string {
 	return splitAndCleanProjectList(raw)
 }
 
+func getExcludeProjectIDsFromOptions(options schema.OptionBlock) []string {
+	raw, ok := options.GetMetadata("exclude_project_ids")
+	if !ok || strings.TrimSpace(raw) == "" {
+		return nil
+	}
+	return splitAndCleanProjectList(raw)
+}
+
 func splitAndCleanProjectList(raw string) []string {
 	replacer := strings.NewReplacer("\n", ",", "\r", ",", ";", ",")
 	normalized := replacer.Replace(raw)

pkg/providers/gcp/project_scope_test.go

@@ -4,6 +4,7 @@ import (
 	"testing"
 
 	assetpb "cloud.google.com/go/asset/apiv1/assetpb"
+	"github.com/projectdiscovery/cloudlist/pkg/schema"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -45,3 +46,62 @@ func TestProjectScopeAllowsAssetByNumber(t *testing.T) {
 
 	assert.True(t, scope.allowsAsset(asset))
 }
+
+func TestGetExcludeProjectIDsFromOptions(t *testing.T) {
+	t.Run("returns nil when not set", func(t *testing.T) {
+		options := schema.OptionBlock{}
+		result := getExcludeProjectIDsFromOptions(options)
+		assert.Nil(t, result)
+	})
+
+	t.Run("parses comma-separated list", func(t *testing.T) {
+		options := schema.OptionBlock{"exclude_project_ids": "project-a,project-b,project-c"}
+		result := getExcludeProjectIDsFromOptions(options)
+		assert.Equal(t, []string{"project-a", "project-b", "project-c"}, result)
+	})
+
+	t.Run("deduplicates entries", func(t *testing.T) {
+		options := schema.OptionBlock{"exclude_project_ids": "project-a,project-a,project-b"}
+		result := getExcludeProjectIDsFromOptions(options)
+		assert.Equal(t, []string{"project-a", "project-b"}, result)
+	})
+}
+
+func TestExcludeProjectValidation(t *testing.T) {
+	t.Run("errors when both include and exclude set", func(t *testing.T) {
+		options := schema.OptionBlock{
+			"provider":            "gcp",
+			"organization_id":     "123456",
+			"project_ids":         "project-a",
+			"exclude_project_ids": "project-b",
+		}
+		_, err := New(options)
+		require.Error(t, err)
+		assert.Contains(t, err.Error(), "mutually exclusive")
+	})
+
+	t.Run("errors when exclude set without organization_id", func(t *testing.T) {
+		options := schema.OptionBlock{
+			"provider":            "gcp",
+			"exclude_project_ids": "project-a",
+		}
+		_, err := New(options)
+		require.Error(t, err)
+		assert.Contains(t, err.Error(), "requires organization_id")
+	})
+}
+
+func TestExcludeProjectFiltering(t *testing.T) {
+	allProjects := []string{"project-a", "project-b", "project-c", "project-d"}
+	excludeScope := newProjectScope([]string{"project-b", "project-d"})
+	require.NotNil(t, excludeScope)
+
+	filtered := make([]string, 0, len(allProjects))
+	for _, p := range allProjects {
+		if !excludeScope.containsID(p) {
+			filtered = append(filtered, p)
+		}
+	}
+
+	assert.Equal(t, []string{"project-a", "project-c"}, filtered)
+}

pkg/schema/schema.go

@@ -240,7 +240,7 @@ func (ob *OptionBlock) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	// Convert raw map to OptionBlock and handle special cases
 	for key, value := range rawMap {
 		switch key {
-		case "account_ids", "exclude_account_ids", "urls", "services", "project_ids":
+		case "account_ids", "exclude_account_ids", "urls", "services", "project_ids", "exclude_project_ids":
 			if valueArr, ok := value.([]interface{}); ok {
 				var strArr []string
 				for _, v := range valueArr {