Harden DNS security, billing consistency and service recovery

.github/workflows/service-regressions.yml

@@ -0,0 +1,55 @@
+name: Service Recovery Regressions
+
+on:
+  pull_request:
+  push:
+    branches:
+      - main
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  service-recovery-regressions:
+    runs-on: ubuntu-24.04
+    timeout-minutes: 20
+    services:
+      postgres:
+        image: postgres:17
+        env:
+          POSTGRES_USER: test
+          POSTGRES_PASSWORD: test
+          POSTGRES_DB: cloud_test
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd "pg_isready -U test -d cloud_test"
+          --health-interval 5s
+          --health-timeout 5s
+          --health-retries 10
+    env:
+      NX_NO_CLOUD: 'true'
+      OBIENTE_TEST_POSTGRES_DSN: 'host=127.0.0.1 port=5432 user=test password=test dbname=cloud_test sslmode=disable'
+    steps:
+      - uses: actions/checkout@v7
+      - uses: pnpm/action-setup@v6
+      - uses: actions/setup-node@v7
+        with:
+          node-version: 22
+          cache: pnpm
+      - uses: actions/setup-go@v7
+        with:
+          go-version: '1.27.3'
+          cache-dependency-path: 'apps/**/go.sum'
+      - run: pnpm install --frozen-lockfile
+      - name: Shared authentication and SMTP regressions
+        env:
+          OBIENTE_TEST_POSTGRES_DSN: ''
+        run: pnpm exec nx run shared:test --skip-nx-cache --command="go test -mod=readonly -race -count=1 ./..."
+      - name: PostgreSQL DNS, billing, organization and email regressions
+        run: pnpm exec nx run-many --target=test --projects=dns-service,billing-service,organizations-service,notifications-service --excludeTaskDependencies --parallel=2 --skip-nx-cache --command="go test -mod=readonly -race -count=1 ./..."

apps/billing-service/internal/service/billing_lock.go

@@ -0,0 +1,42 @@
+package billing
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/obiente/cloud/apps/shared/pkg/database"
+	"gorm.io/gorm"
+)
+
+// Lock and billing queries share one transaction and one pool connection.
+// Transaction-scoped locks are released on rollback, cancellation or commit.
+func withBillingLock(ctx context.Context, orgID string, operation func(*gorm.DB) error) error {
+	return database.DB.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
+		if tx.Dialector.Name() == "postgres" {
+			if err := tx.Exec("SELECT pg_advisory_xact_lock(hashtextextended(?, 0))", "monthly-billing:"+orgID).Error; err != nil {
+				return err
+			}
+		}
+		return operation(tx)
+	})
+}
+
+func billingMetricsDB(tx *gorm.DB) (*gorm.DB, error) {
+	metrics := database.GetMetricsDB()
+	if metrics == nil {
+		return nil, fmt.Errorf("metrics database not available")
+	}
+	metricsPool, err := metrics.DB()
+	if err != nil {
+		return nil, err
+	}
+	applicationPool, err := database.DB.DB()
+	if err != nil {
+		return nil, err
+	}
+	if metricsPool == applicationPool {
+		// Self-hosted setups may share the primary database for metrics.
+		return tx, nil
+	}
+	return metrics.WithContext(tx.Statement.Context), nil
+}

apps/billing-service/internal/service/billing_lock_test.go

@@ -0,0 +1,120 @@
+package billing
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/obiente/cloud/apps/shared/pkg/database"
+	"gorm.io/driver/postgres"
+	"gorm.io/gorm"
+)
+
+func TestBillingCompletesWithSingleConnectionPool(t *testing.T) {
+	for _, early := range []bool{false, true} {
+		t.Run(fmt.Sprintf("early=%v", early), func(t *testing.T) {
+			db := monthlyRecoveryDB(t)
+			now := time.Now().UTC()
+			seedOverdueAccount(t, db, now.AddDate(0, -2, 0), 1)
+			pool, err := db.DB()
+			if err != nil {
+				t.Fatal(err)
+			}
+			pool.SetMaxOpenConns(1)
+			ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+			defer cancel()
+			err = withBillingLock(ctx, "org-synthetic", func(tx *gorm.DB) error {
+				if early {
+					_, _, err := generateCurrentBillEarlyLocked(tx, "org-synthetic")
+					return err
+				}
+				return processOrganizationBillingLocked(tx, "org-synthetic", now)
+			})
+			if err != nil {
+				t.Fatal(err)
+			}
+			var count int64
+			if err := db.Model(&database.MonthlyBill{}).Count(&count).Error; err != nil || count != 1 {
+				t.Fatalf("bills=%d err=%v", count, err)
+			}
+		})
+	}
+}
+
+func TestConcurrentBillingDoesNotExhaustPool(t *testing.T) {
+	db := monthlyRecoveryDB(t)
+	now := time.Now().UTC()
+	day := 1
+	for i := range 4 {
+		id := fmt.Sprintf("org-%d", i)
+		if err := db.Create(&database.Organization{ID: id, Slug: id, Status: "active", CreatedAt: now.AddDate(0, -2, 0)}).Error; err != nil {
+			t.Fatal(err)
+		}
+		if err := db.Create(&database.BillingAccount{ID: fmt.Sprintf("account-%d", i), OrganizationID: id, Status: "ACTIVE", BillingDate: &day}).Error; err != nil {
+			t.Fatal(err)
+		}
+	}
+	pool, _ := db.DB()
+	pool.SetMaxOpenConns(2)
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	results := make(chan error, 4)
+	for i := range 4 {
+		go func(i int) {
+			id := fmt.Sprintf("org-%d", i)
+			results <- withBillingLock(ctx, id, func(tx *gorm.DB) error { _, _, err := generateCurrentBillEarlyLocked(tx, id); return err })
+		}(i)
+	}
+	for range 4 {
+		if err := <-results; err != nil {
+			t.Error(err)
+		}
+	}
+	var count int64
+	if err := db.Model(&database.MonthlyBill{}).Count(&count).Error; err != nil || count != 4 {
+		t.Fatalf("bills=%d err=%v", count, err)
+	}
+}
+
+func TestBillingCancellationReleasesConnectionAndLock(t *testing.T) {
+	db := monthlyRecoveryDB(t)
+	pool, _ := db.DB()
+	pool.SetMaxOpenConns(1)
+	ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
+	defer cancel()
+	err := withBillingLock(ctx, "org-synthetic", func(tx *gorm.DB) error { return tx.Exec("SELECT pg_sleep(10)").Error })
+	if err == nil || ctx.Err() == nil {
+		t.Fatalf("query ignored cancellation: %v", err)
+	}
+	retry, cancelRetry := context.WithTimeout(context.Background(), 2*time.Second)
+	defer cancelRetry()
+	if err := withBillingLock(retry, "org-synthetic", func(tx *gorm.DB) error { return tx.Exec("SELECT 1").Error }); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestSeparateMetricsQueriesInheritBillingDeadline(t *testing.T) {
+	db := monthlyRecoveryDB(t)
+	previous := database.MetricsDB
+	// A separate pool for the same fixture represents a separate metrics server.
+	dsn := db.Dialector.(*postgres.Dialector).Config.DSN
+	metrics, err := gorm.Open(postgres.Open(dsn), &gorm.Config{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	database.MetricsDB = metrics
+	t.Cleanup(func() { database.MetricsDB = previous; pool, _ := metrics.DB(); pool.Close() })
+	ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
+	defer cancel()
+	err = withBillingLock(ctx, "org-synthetic", func(tx *gorm.DB) error {
+		metrics, err := billingMetricsDB(tx)
+		if err != nil {
+			return err
+		}
+		return metrics.Exec("SELECT pg_sleep(10)").Error
+	})
+	if err == nil || ctx.Err() == nil {
+		t.Fatalf("metrics ignored deadline: %v", err)
+	}
+}