Integration tests improvements

Makefile

@@ -172,7 +172,17 @@ integration-test: $(SELECTED_PACKAGE) build-mock-management-plane-grpc
 	TEST_ENV="Container" CONTAINER_OS_TYPE=$(CONTAINER_OS_TYPE) BUILD_TARGET="install-agent-local" CONTAINER_NGINX_IMAGE_REGISTRY=${CONTAINER_NGINX_IMAGE_REGISTRY} \
 	PACKAGES_REPO=$(OSS_PACKAGES_REPO) PACKAGE_NAME=$(PACKAGE_NAME) BASE_IMAGE=$(BASE_IMAGE) DOCKERFILE_PATH=$(DOCKERFILE_PATH) IMAGE_PATH=$(IMAGE_PATH) TAG=${IMAGE_TAG} \
 	OS_VERSION=$(OS_VERSION) OS_RELEASE=$(OS_RELEASE) \
-	go test -v ./test/integration/installuninstall ./test/integration/managementplane ./test/integration/auxiliarycommandserver ./test/integration/nginxless
+	go test -v -parallel 2 ./test/integration/installuninstall ./test/integration/nginxless
+
+	TEST_ENV="Container" CONTAINER_OS_TYPE=$(CONTAINER_OS_TYPE) BUILD_TARGET="install-agent-local" CONTAINER_NGINX_IMAGE_REGISTRY=${CONTAINER_NGINX_IMAGE_REGISTRY} \
+	PACKAGES_REPO=$(OSS_PACKAGES_REPO) PACKAGE_NAME=$(PACKAGE_NAME) BASE_IMAGE=$(BASE_IMAGE) DOCKERFILE_PATH=$(DOCKERFILE_PATH) IMAGE_PATH=$(IMAGE_PATH) TAG=${IMAGE_TAG} \
+	OS_VERSION=$(OS_VERSION) OS_RELEASE=$(OS_RELEASE) \
+	go test -v ./test/integration/managementplane
+
+	TEST_ENV="Container" CONTAINER_OS_TYPE=$(CONTAINER_OS_TYPE) BUILD_TARGET="install-agent-local" CONTAINER_NGINX_IMAGE_REGISTRY=${CONTAINER_NGINX_IMAGE_REGISTRY} \
+	PACKAGES_REPO=$(OSS_PACKAGES_REPO) PACKAGE_NAME=$(PACKAGE_NAME) BASE_IMAGE=$(BASE_IMAGE) DOCKERFILE_PATH=$(DOCKERFILE_PATH) IMAGE_PATH=$(IMAGE_PATH) TAG=${IMAGE_TAG} \
+	OS_VERSION=$(OS_VERSION) OS_RELEASE=$(OS_RELEASE) \
+	go test -v ./test/integration/auxiliarycommandserver
 
 upgrade-test: $(SELECTED_PACKAGE) build-mock-management-plane-grpc
 	TEST_ENV="Container" CONTAINER_OS_TYPE=$(CONTAINER_OS_TYPE) BUILD_TARGET="install-agent-repo" CONTAINER_NGINX_IMAGE_REGISTRY=${CONTAINER_NGINX_IMAGE_REGISTRY} \

test/helpers/test_containers_utils.go

@@ -11,6 +11,7 @@ import (
 	"io"
 	"os"
 	"testing"
+	"time"
 
 	"github.com/moby/moby/client"
 	"github.com/stretchr/testify/assert"
@@ -21,6 +22,11 @@ import (
 
 const configFilePermissions = 0o600
 
+const (
+	extractFileMaxAttempts = 10
+	extractFileRetryDelay  = 100 * time.Millisecond
+)
+
 type Parameters struct {
 	NginxConfigPath          string
 	NginxAgentConfigPath     string
@@ -496,8 +502,33 @@ func ExtractFileFromContainer(
 	containerPath string,
 ) string {
 	tb.Helper()
-	fileContent, err := testContainer.CopyFileFromContainer(ctx, containerPath)
+
+	var (
+		fileContent io.ReadCloser
+		err         error
+	)
+
+	for attempt := 1; attempt <= extractFileMaxAttempts; attempt++ {
+		fileContent, err = testContainer.CopyFileFromContainer(ctx, containerPath)
+		if err == nil {
+			break
+		}
+
+		if attempt == extractFileMaxAttempts {
+			break
+		}
+
+		select {
+		case <-ctx.Done():
+			require.NoError(tb, ctx.Err())
+		case <-time.After(extractFileRetryDelay):
+		}
+	}
+
 	require.NoError(tb, err)
+	defer func() {
+		require.NoError(tb, fileContent.Close())
+	}()
 
 	content, err := io.ReadAll(fileContent)
 	require.NoError(tb, err)

test/mock/grpc/mock_management_command_service.go

@@ -43,6 +43,7 @@ type CommandService struct {
 	externalFileServer           string
 	configDirectory              string
 	dataPlaneResponses           []*mpi.DataPlaneResponse
+	instanceFilesMutex           sync.RWMutex
 	dataPlaneResponsesMutex      sync.Mutex
 	updateDataPlaneStatusMutex   sync.Mutex
 	connectionMutex              sync.Mutex
@@ -216,8 +217,11 @@ func (cs *CommandService) handleConfigUploadRequest(
 	instanceID := upload.ConfigUploadRequest.GetOverview().GetConfigVersion().GetInstanceId()
 	overviewFiles := upload.ConfigUploadRequest.GetOverview().GetFiles()
 
+	cs.instanceFilesMutex.Lock()
+	defer cs.instanceFilesMutex.Unlock()
+
 	if cs.instanceFiles[instanceID] != nil {
-		filesToDelete := cs.checkForDeletedFiles(instanceID, overviewFiles)
+		filesToDelete := cs.checkForDeletedFilesLocked(instanceID, overviewFiles)
 		for _, fileToDelete := range filesToDelete {
 			err := os.Remove(fileToDelete)
 			if err != nil {
@@ -228,7 +232,8 @@ func (cs *CommandService) handleConfigUploadRequest(
 	cs.instanceFiles[instanceID] = overviewFiles
 }
 
-func (cs *CommandService) checkForDeletedFiles(instanceID string, overviewFiles []*mpi.File) []string {
+// checkForDeletedFilesLocked must be called while holding instanceFilesMutex
+func (cs *CommandService) checkForDeletedFilesLocked(instanceID string, overviewFiles []*mpi.File) []string {
 	filesToDelete := []string{}
 
 	for _, diskfile := range cs.instanceFiles[instanceID] {
@@ -401,6 +406,7 @@ func (cs *CommandService) addConfigApplyEndpoint() {
 			return
 		}
 
+		cs.instanceFilesMutex.Lock()
 		if filesUpdated {
 			cs.instanceFiles[instanceID] = updatedConfigFiles
 		} else {
@@ -425,6 +431,7 @@ func (cs *CommandService) addConfigApplyEndpoint() {
 				},
 			},
 		}
+		cs.instanceFilesMutex.Unlock()
 
 		cs.requestChan <- &request
 
@@ -437,6 +444,7 @@ func (cs *CommandService) addConfigEndpoint() {
 		instanceID := c.Param("instanceID")
 		var data map[string]interface{}
 
+		cs.instanceFilesMutex.RLock()
 		response := &mpi.GetOverviewResponse{
 			Overview: &mpi.FileOverview{
 				ConfigVersion: &mpi.ConfigVersion{
@@ -446,6 +454,7 @@ func (cs *CommandService) addConfigEndpoint() {
 				Files: cs.instanceFiles[instanceID],
 			},
 		}
+		cs.instanceFilesMutex.RUnlock()
 
 		if err := json.Unmarshal([]byte(protojson.Format(response)), &data); err != nil {
 			slog.Error("Failed to return connection", "error", err)

test/mock/grpc/mock_management_server.go

@@ -18,7 +18,7 @@ import (
 	"syscall"
 	"time"
 
-	"github.com/nginx/agent/v3/api/grpc/mpi/v1"
+	mpi "github.com/nginx/agent/v3/api/grpc/mpi/v1"
 	"github.com/nginx/agent/v3/internal/config"
 
 	"buf.build/go/protovalidate"
@@ -39,6 +39,7 @@ const (
 	keepAliveTimeout = 10 * time.Second
 	testTimeout      = 100 * time.Millisecond
 	connectionType   = "tcp"
+	requestChanSize  = 100
 )
 
 var (
@@ -71,7 +72,7 @@ func NewMockManagementServer(
 	externalFileServer *string,
 ) (*MockManagementServer, error) {
 	var err error
-	requestChan := make(chan *v1.ManagementPlaneRequest)
+	requestChan := make(chan *mpi.ManagementPlaneRequest, requestChanSize)
 
 	commandService := serveCommandService(ctx, apiAddress, agentConfig, requestChan, *configDirectory,
 		*externalFileServer)
@@ -98,8 +99,8 @@ func NewMockManagementServer(
 	healthcheck := health.NewServer()
 	healthgrpc.RegisterHealthServer(grpcServer, healthcheck)
 
-	v1.RegisterCommandServiceServer(grpcServer, commandService)
-	v1.RegisterFileServiceServer(grpcServer, fileServer)
+	mpi.RegisterCommandServiceServer(grpcServer, commandService)
+	mpi.RegisterFileServiceServer(grpcServer, fileServer)
 	go reportHealth(healthcheck, agentConfig)
 
 	go func() {
@@ -187,7 +188,7 @@ func serveCommandService(
 	ctx context.Context,
 	apiAddress string,
 	agentConfig *config.Config,
-	requestChan chan *v1.ManagementPlaneRequest,
+	requestChan chan *mpi.ManagementPlaneRequest,
 	configDirectory string,
 	externalFileServer string,
 ) *CommandService {