CMP-4485: [experiment] Parallelize destructive serial e2e tests across per-worker pools

Makefile

@@ -621,8 +621,11 @@ e2e-deployment: e2e-set-image prep-e2e ## Run operator deployment end-to-end tes
 	@CONTENT_IMAGE=$(E2E_CONTENT_IMAGE_PATH) BROKEN_CONTENT_IMAGE=$(E2E_BROKEN_CONTENT_IMAGE_PATH) $(GO) test ./tests/e2e/deployment $(E2E_GO_TEST_FLAGS) -args $(E2E_ARGS) | tee tests/e2e-deployment-test.log
 
 .PHONY: e2e-serial
-e2e-serial: e2e-set-image prep-e2e ## Run destructive end-to-end tests serially.
-	@CONTENT_IMAGE=$(E2E_CONTENT_IMAGE_PATH) BROKEN_CONTENT_IMAGE=$(E2E_BROKEN_CONTENT_IMAGE_PATH) $(GO) test ./tests/e2e/serial $(E2E_GO_TEST_FLAGS) -args $(E2E_ARGS) | tee tests/e2e-serial.log
+# Number of isolated MachineConfigPool lanes the destructive/reboot tests run
+# across in parallel (one worker node per lane). Capped at the worker count.
+E2E_PARALLEL_POOLS?=3
+e2e-serial: e2e-set-image prep-e2e ## Run destructive end-to-end tests, sharded across E2E_PARALLEL_POOLS pools in parallel.
+	@CONTENT_IMAGE=$(E2E_CONTENT_IMAGE_PATH) BROKEN_CONTENT_IMAGE=$(E2E_BROKEN_CONTENT_IMAGE_PATH) E2E_PARALLEL_POOLS=$(E2E_PARALLEL_POOLS) $(GO) test ./tests/e2e/serial $(E2E_GO_TEST_FLAGS) -parallel $(E2E_PARALLEL_POOLS) -args $(E2E_ARGS) | tee tests/e2e-serial.log
 
 .PHONY: e2e-tailoring
 e2e-tailoring: e2e-set-image prep-e2e ## Run profile tailoring end-to-end tests.

tests/e2e/framework/common.go

@@ -659,6 +659,13 @@ func (f *Framework) updateScanSettingsForDebug() error {
 }
 
 func (f *Framework) ensureE2EScanSettings() error {
+	return f.ensureE2EScanSettingsForPool(TestPoolName)
+}
+
+// ensureE2EScanSettingsForPool creates a "<poolName>-default" and
+// "<poolName>-default-auto-apply" ScanSetting whose Roles target poolName, so
+// each parallel test lane scans/remediates only its own pool.
+func (f *Framework) ensureE2EScanSettingsForPool(poolName string) error {
 	if f.Platform == "rosa" {
 		fmt.Printf("bypassing ScanSettings test setup because it's not supported on %s\n", f.Platform)
 		return nil
@@ -672,11 +679,11 @@ func (f *Framework) ensureE2EScanSettings() error {
 
 		ssCopy := ss.DeepCopy()
 		ssCopy.ObjectMeta = metav1.ObjectMeta{
-			Name:      "e2e-" + ssName,
+			Name:      poolName + "-" + ssName,
 			Namespace: f.OperatorNamespace,
 		}
 		ssCopy.Roles = []string{
-			TestPoolName,
+			poolName,
 		}
 		ssCopy.Debug = true
 
@@ -706,12 +713,8 @@ func (f *Framework) deleteScanSettings(name string) error {
 	return nil
 }
 
-func (f *Framework) createMachineConfigPool(n string) error {
-	if f.Platform == "rosa" {
-		fmt.Printf("bypassing MachineConfigPool test setup because it's not supported on %s\n", f.Platform)
-		return nil
-	}
-	// get the worker pool
+// getWorkerNodes returns the nodes currently in the worker MachineConfigPool.
+func (f *Framework) getWorkerNodes() ([]corev1.Node, error) {
 	w := "worker"
 	p := &mcfgv1.MachineConfigPool{}
 	getErr := backoff.RetryNotify(
@@ -729,15 +732,42 @@ func (f *Framework) createMachineConfigPool(n string) error {
 			log.Printf("error while getting MachineConfig pool to create sub-pool from: %s. Retrying after %s", err, interval)
 		})
 	if getErr != nil {
-		return fmt.Errorf("failed to get Machine Config Pool %s to create sub-pool from: %w", w, getErr)
+		return nil, fmt.Errorf("failed to get Machine Config Pool %s to create sub-pool from: %w", w, getErr)
 	}
-
 	nodeList, err := f.getNodesForPool(p)
+	if err != nil {
+		return nil, err
+	}
+	return nodeList.Items, nil
+}
+
+func (f *Framework) createMachineConfigPool(n string) error {
+	if f.Platform == "rosa" {
+		fmt.Printf("bypassing MachineConfigPool test setup because it's not supported on %s\n", f.Platform)
+		return nil
+	}
+	nodes, err := f.getWorkerNodes()
 	if err != nil {
 		return err
 	}
+	if len(nodes) == 0 {
+		return fmt.Errorf("no worker nodes found to create Machine Config Pool %s from", n)
+	}
 	// pick the first node in the list so we only have a pool of one
-	node := nodeList.Items[0]
+	return f.createMachineConfigPoolFromNode(n, &nodes[0])
+}
+
+// createMachineConfigPoolFromNode creates a MachineConfigPool named n containing
+// exactly the given node (relabeled into the pool's role). This lets several
+// isolated pools be created from distinct worker nodes so destructive tests can
+// run in parallel, one node per pool.
+func (f *Framework) createMachineConfigPoolFromNode(n string, node *corev1.Node) error {
+	if f.Platform == "rosa" {
+		fmt.Printf("bypassing MachineConfigPool test setup because it's not supported on %s\n", f.Platform)
+		return nil
+	}
+	// the base pool the sub-pool inherits MachineConfigs from
+	w := "worker"
 
 	// create a new pool with a subset of the nodes
 	l := fmt.Sprintf("node-role.kubernetes.io/%s", n)
@@ -763,7 +793,7 @@ func (f *Framework) createMachineConfigPool(n string) error {
 	nodeLabel := make(map[string]string)
 	nodeLabel[l] = ""
 	poolLabels := make(map[string]string)
-	poolLabels["pools.operator.machineconfiguration.openshift.io/e2e"] = ""
+	poolLabels["pools.operator.machineconfiguration.openshift.io/"+n] = ""
 	newPool := &mcfgv1.MachineConfigPool{
 		ObjectMeta: metav1.ObjectMeta{Name: n, Labels: poolLabels},
 		Spec: mcfgv1.MachineConfigPoolSpec{
@@ -801,7 +831,7 @@ func (f *Framework) createMachineConfigPool(n string) error {
 	}
 
 	// wait for pool to come up
-	err = wait.PollImmediate(machineOperationRetryInterval, machineOperationTimeout, func() (bool, error) {
+	err := wait.PollImmediate(machineOperationRetryInterval, machineOperationTimeout, func() (bool, error) {
 		pool := mcfgv1.MachineConfigPool{}
 		err := f.Client.Get(context.TODO(), types.NamespacedName{Name: n}, &pool)
 		if err != nil {

tests/e2e/framework/framework.go

@@ -62,6 +62,12 @@ type Framework struct {
 	WatchNamespace    string
 	Platform          string
 
+	// TestPools hands out isolated MachineConfigPool lanes to destructive tests
+	// that run in parallel (see testpools.go). testPoolNames tracks the lanes
+	// created during setup so they can be cleaned up on teardown.
+	TestPools     chan *TestPool
+	testPoolNames []string
+
 	restMapper *restmapper.DeferredDiscoveryRESTMapper
 
 	projectRoot       string

tests/e2e/framework/main_entry.go

@@ -116,14 +116,12 @@ func (f *Framework) SetUp() error {
 		return fmt.Errorf("failed to set scan setting bindings to debug: %w", err)
 	}
 
-	err = f.ensureE2EScanSettings()
+	// Create one isolated MachineConfigPool lane per worker node (plus matching
+	// ScanSettings) so the destructive/reboot tests can run in parallel, each
+	// against its own pool and node. See testpools.go.
+	err = f.setUpTestPools()
 	if err != nil {
-		return fmt.Errorf("failed to configure scan settings for tests: %w", err)
-	}
-
-	err = f.createMachineConfigPool("e2e")
-	if err != nil {
-		return fmt.Errorf("failed to create Machine Config Pool %s: %w", "e2e", err)
+		return fmt.Errorf("failed to set up test pools: %w", err)
 	}
 	err = f.createInvalidMachineConfigPool("e2e-invalid")
 	if err != nil {
@@ -162,20 +160,11 @@ func (f *Framework) TearDown() error {
 	if os.Getenv("SKIP_MCP_SETUP") != "" {
 		log.Println("SKIP_MCP_SETUP is set, skipping e2e ScanSettings and MachineConfigPool cleanup")
 	} else {
-		err = f.deleteScanSettings("e2e-default")
-		if err != nil {
-			return err
-		}
-		err = f.deleteScanSettings("e2e-default-auto-apply")
-		if err != nil {
-			return err
-		}
-		// unlabel nodes
-		err = f.restoreNodeLabelsForPool("e2e")
-		if err != nil {
-			return err
-		}
-		err = f.cleanUpMachineConfigPool("e2e")
+		// Clean up per-lane ScanSettings. We deliberately don't restore node
+		// labels or delete the lane MachineConfigPools here (see tearDownTestPools):
+		// that would reboot every lane node back to rendered-worker, and the CI
+		// cluster is torn down right after, so the work would be wasted.
+		err = f.tearDownTestPools()
 		if err != nil {
 			return err
 		}

tests/e2e/framework/testpools.go

@@ -0,0 +1,130 @@
+package framework
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"strconv"
+	"testing"
+
+	"github.com/ComplianceAsCode/compliance-operator/pkg/utils"
+)
+
+// TestPool is one isolated MachineConfigPool "lane" that a destructive serial
+// test runs against. Each lane owns a distinct worker node and its own pair of
+// ScanSettings, so the reboot-heavy tests can execute in parallel without
+// contending on a single shared pool/node.
+type TestPool struct {
+	Index                int
+	Name                 string
+	DefaultScanSetting   string
+	AutoApplyScanSetting string
+}
+
+// NodeRoleSelector returns the node selector matching this lane's single node.
+func (p *TestPool) NodeRoleSelector() map[string]string {
+	return utils.GetNodeRoleSelector(p.Name)
+}
+
+// testPoolCount is the number of parallel destructive lanes to set up. It
+// defaults to 1 (a single "e2e" pool, identical to the historical behavior, so
+// non-serial suites that share SetUp are unaffected) and is raised to N via
+// E2E_PARALLEL_POOLS - the serial suite's Makefile target sets it. setUpTestPools
+// caps it at the number of available worker nodes.
+func testPoolCount() int {
+	if v := os.Getenv("E2E_PARALLEL_POOLS"); v != "" {
+		if n, err := strconv.Atoi(v); err == nil && n > 0 {
+			return n
+		}
+		log.Printf("ignoring invalid E2E_PARALLEL_POOLS=%q; using default", v)
+	}
+	return 1
+}
+
+// setUpTestPools carves one MachineConfigPool lane per worker node (up to
+// testPoolCount) and creates a matching pair of ScanSettings for each. Lanes are
+// handed to tests via AcquireTestPool. We reuse the existing worker nodes rather
+// than scaling the cluster; when every worker becomes a lane the operator still
+// runs (nodes keep their worker label) but has no idle worker to fall back to
+// during simultaneous reboots.
+func (f *Framework) setUpTestPools() error {
+	if f.Platform == "rosa" {
+		fmt.Printf("bypassing test pool setup because MachineConfigPools are not supported on %s\n", f.Platform)
+		f.TestPools = make(chan *TestPool, 1)
+		return nil
+	}
+
+	nodes, err := f.getWorkerNodes()
+	if err != nil {
+		return fmt.Errorf("failed to list worker nodes for test pools: %w", err)
+	}
+
+	n := testPoolCount()
+	if n > len(nodes) {
+		log.Printf("E2E_PARALLEL_POOLS=%d exceeds available worker nodes (%d); capping to %d", n, len(nodes), len(nodes))
+		n = len(nodes)
+	}
+	if n < 1 {
+		return fmt.Errorf("no worker nodes available to create test pools")
+	}
+
+	f.TestPools = make(chan *TestPool, n)
+	f.testPoolNames = nil
+	for i := 0; i < n; i++ {
+		name := fmt.Sprintf("%s-%d", TestPoolName, i)
+		if n == 1 {
+			// Preserve the historical single-pool name "e2e" (and "e2e-default"
+			// ScanSettings) when not sharding, so suites that share SetUp but
+			// don't run in parallel behave exactly as before.
+			name = TestPoolName
+		}
+		node := &nodes[i]
+		if err := f.createMachineConfigPoolFromNode(name, node); err != nil {
+			return fmt.Errorf("failed to create test pool %s: %w", name, err)
+		}
+		if err := f.ensureE2EScanSettingsForPool(name); err != nil {
+			return fmt.Errorf("failed to create scan settings for test pool %s: %w", name, err)
+		}
+		f.testPoolNames = append(f.testPoolNames, name)
+		f.TestPools <- &TestPool{
+			Index:                i,
+			Name:                 name,
+			DefaultScanSetting:   name + "-default",
+			AutoApplyScanSetting: name + "-default-auto-apply",
+		}
+		log.Printf("test pool lane %d ready on node %s: %s", i, node.Name, name)
+	}
+	return nil
+}
+
+// AcquireTestPool checks out an isolated pool lane for a destructive test,
+// blocking until one is free, and returns it when the test ends. Call
+// t.Parallel() before this so lanes are shared across concurrent tests.
+func (f *Framework) AcquireTestPool(t *testing.T) *TestPool {
+	t.Helper()
+	p := <-f.TestPools
+	t.Logf("acquired test pool lane %s", p.Name)
+	t.Cleanup(func() {
+		f.TestPools <- p
+		t.Logf("released test pool lane %s", p.Name)
+	})
+	return p
+}
+
+// tearDownTestPools deletes the per-lane ScanSettings. It intentionally does NOT
+// restore node labels or delete the MachineConfigPools: that would reboot every
+// lane node back to rendered-worker, and the CI cluster is destroyed right after
+// the run, so the work would be wasted.
+func (f *Framework) tearDownTestPools() error {
+	if f.Platform == "rosa" {
+		return nil
+	}
+	for _, name := range f.testPoolNames {
+		for _, suffix := range []string{"-default", "-default-auto-apply"} {
+			if err := f.deleteScanSettings(name + suffix); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}