feat: Add cloud storage integration with S3 upload/download and snapshot management

.gitignore

@@ -1,6 +1,6 @@
 .DS_Store
 /.idea/
-/bin/
+**/bin/
 data-dir/
 output/
 .vercel
@@ -10,3 +10,12 @@ contracts/cache/
 contracts/out/
 contracts/abi/
 benchmark/benchmark
+
+.env
+
+# Node
+coverage/
+!**/__fixtures__/coverage
+node_modules
+yarn-debug.log*
+yarn-error.log*

Makefile

@@ -19,6 +19,9 @@ LDFLAGSSTRING +=-X main.GitDate=$(GITDATE)
 LDFLAGSSTRING +=-X main.Version=$(VERSION)
 LDFLAGS := -ldflags "$(LDFLAGSSTRING)"
 
+# Include .env file if it exists
+-include .env
+
 # first so that make defaults to building the benchmark
 .PHONY: build
 build:
@@ -50,3 +53,11 @@ build-rbuilder:
 
 .PHONY: build-binaries
 build-binaries: build-reth build-geth build-rbuilder
+
+.PHONY: build-frontend
+build-frontend:
+	cd report && yarn build
+
+.PHONY: run-frontend
+run-frontend:
+	cd report && set -a && [ -f ../.env ] && . ../.env && set +a && yarn dev

README.md

@@ -31,6 +31,9 @@ git submodule update --init --recursive
 # Build the application
 make build
 
+# Build the binaries, geth, reth, rbuilder
+make build-binaries
+
 # Run the basic benchmark
 ./bin/base-bench run \
   --config ./configs/public/basic.yml \

SNAPSHOT_SYSTEM.md

@@ -0,0 +1,97 @@
+# Two-Tier Snapshot System
+
+## Overview
+
+The benchmarking framework now supports an optimized two-tier snapshot system that significantly improves performance and reduces network overhead when running multiple tests with snapshots.
+
+## Architecture
+
+### Tier 1: Initial Snapshots
+- **Purpose**: Downloaded once at benchmark startup and stored persistently
+- **Location**: Typically `/data/snapshots/initial_<nodeType>_<hash>`
+- **Lifecycle**: Created at benchmark startup, persisted across all tests
+- **Usage**: Serves as the source for per-test copies
+
+### Tier 2: Per-Test Snapshots  
+- **Purpose**: Test-specific copies created from initial snapshots
+- **Location**: Test-specific temporary directories
+- **Lifecycle**: Created before each test, cleaned up after test completion
+- **Usage**: Isolated environment for each test run
+
+## Configuration Format
+
+### New YAML Structure
+
+```yaml
+benchmarks:
+  - initial_snapshots:
+      - node_type: reth
+        command: ./scripts/setup-initial-snapshot.sh --network=sepolia --node-type=reth
+        superchain_chain_id: 84532
+      - node_type: geth
+        command: ./scripts/setup-initial-snapshot.sh --network=sepolia --node-type=geth  
+        superchain_chain_id: 84532
+    variables:
+      - type: node_type
+        values: [reth, geth]
+      # ... other variables
+```
+
+## Implementation Details
+
+### Key Components
+
+1. **SnapshotManager Interface** (`benchmark/snapshots.go`)
+   - `EnsureInitialSnapshot()`: Creates initial snapshots
+   - `GetInitialSnapshotPath()`: Retrieves initial snapshot paths
+   - `CopyFromInitialSnapshot()`: Copies using rsync for efficiency
+
+2. **TestDefinition** (`benchmark/definition.go`)
+   - `InitialSnapshots []SnapshotDefinition`: Tier 1 snapshots
+
+3. **Service** (`runner/service.go`)
+   - `setupInitialSnapshots()`: Runs at benchmark startup
+   - `setupInternalDirectories()`: Uses rsync for per-test copies
+
+### Execution Flow
+
+1. **Benchmark Startup**
+   ```
+   Service.Run() → setupInitialSnapshots() → EnsureInitialSnapshot()
+   ```
+
+2. **Per Test Execution**
+   ```
+   runTest() → setupInternalDirectories() → CopyFromInitialSnapshot()
+   ```
+
+3. **Test Cleanup**
+   ```
+   defer cleanup → os.RemoveAll(testDir) // Removes per-test copies only
+   ```
+
+## Usage Examples
+
+### Multi-Node Type Testing
+```yaml
+initial_snapshots:
+  - node_type: reth
+    command: ./download-reth-snapshot.sh
+  - node_type: geth  
+    command: ./download-geth-snapshot.sh
+variables:
+  - type: node_type
+    values: [reth, geth]
+```
+
+### Fallback Support
+If no initial snapshot exists for a node type, the system automatically falls back to the original single-tier behavior, ensuring backward compatibility.
+
+### Mixed Scenarios
+```yaml
+# Test plan 1: Uses two-tier system
+- initial_snapshots: [...]
+
+# Test plan 2: Uses single-tier system  
+- initial_snapshots: [...]  # No initial_snapshots
+```

benchmark/cmd/main.go

@@ -42,8 +42,8 @@ func main() {
 			Name:        "import-runs",
 			Flags:       cliapp.ProtectFlags(flags.ImportRunsFlags),
 			Action:      ImportMain(Version),
-			Usage:       "import runs from metadata file or URL",
-			Description: "Import benchmark runs from local metadata.json or remote URL into existing output metadata.json. Use --src-tag and --dest-tag to apply tags to runs, or use interactive mode.",
+			Usage:       "import runs from metadata file, URL",
+			Description: "Import benchmark runs from local metadata.json, remote URL into existing output metadata.json. Use --src-tag and --dest-tag to apply tags to runs, or use interactive mode.",
 			ArgsUsage:   "[metadata-file-or-url]",
 		},
 	}
@@ -86,7 +86,7 @@ func ImportMain(version string) cli.ActionFunc {
 
 		service := importer.NewService(cfg, l)
 
-		// Load source metadata
+		// Load source metadata from file/URL
 		srcMetadata, err := service.LoadSourceMetadata(cfg.SourceFile())
 		if err != nil {
 			return fmt.Errorf("failed to load source metadata: %w", err)

benchmark/flags/flags.go

@@ -15,11 +15,16 @@ func prefixEnvVars(name string) []string {
 }
 
 const (
-	ConfigFlagName    = "config"
-	RootDirFlagName   = "root-dir"
-	OutputDirFlagName = "output-dir"
-	TxFuzzBinFlagName = "tx-fuzz-bin"
-	ProxyPortFlagName = "proxy-port"
+	ConfigFlagName          = "config"
+	RootDirFlagName         = "root-dir"
+	OutputDirFlagName       = "output-dir"
+	TxFuzzBinFlagName       = "tx-fuzz-bin"
+	ProxyPortFlagName       = "proxy-port"
+	BenchmarkRunIDFlagName  = "benchmark-run-id"
+	MachineTypeFlagName     = "machine-type"
+	MachineProviderFlagName = "machine-provider"
+	MachineRegionFlagName   = "machine-region"
+	FileSystemFlagName      = "file-system"
 )
 
 // TxFuzz defaults
@@ -62,6 +67,37 @@ var (
 		Value:   8546,
 		EnvVars: prefixEnvVars("PROXY_PORT"),
 	}
+
+	BenchmarkRunIDFlag = &cli.StringFlag{
+		Name:    BenchmarkRunIDFlagName,
+		Usage:   "Custom benchmark run ID (auto-generated if not provided)",
+		EnvVars: prefixEnvVars("BENCHMARK_RUN_ID"),
+	}
+
+	MachineTypeFlag = &cli.StringFlag{
+		Name:    "machine-type",
+		Usage:   "Machine type (e.g., i4i.32xlarge)",
+		EnvVars: prefixEnvVars("MACHINE_TYPE"),
+	}
+
+	MachineProviderFlag = &cli.StringFlag{
+		Name:    "machine-provider",
+		Usage:   "Cloud provider (aws or gcp)",
+		EnvVars: prefixEnvVars("MACHINE_PROVIDER"),
+	}
+
+	MachineRegionFlag = &cli.StringFlag{
+		Name:    "machine-region",
+		Usage:   "Machine region",
+		EnvVars: prefixEnvVars("MACHINE_REGION"),
+	}
+
+	FileSystemFlag = &cli.StringFlag{
+		Name:    "file-system",
+		Usage:   "File system type",
+		Value:   "ext4",
+		EnvVars: prefixEnvVars("FILE_SYSTEM"),
+	}
 )
 
 // Flags contains the list of configuration options available to the binary.
@@ -73,6 +109,11 @@ var RunFlags = []cli.Flag{
 	OutputDirFlag,
 	TxFuzzBinFlag,
 	ProxyPortFlag,
+	BenchmarkRunIDFlag,
+	MachineTypeFlag,
+	MachineProviderFlag,
+	MachineRegionFlag,
+	FileSystemFlag,
 }
 
 func init() {

clients/build-reth.sh

@@ -40,11 +40,11 @@ git checkout "$RETH_VERSION"
 
 # Build the binary using cargo
 echo "Building reth with cargo..."
-cargo build --release --bin reth
+cargo build --bin op-reth --profile maxperf --manifest-path crates/optimism/bin/Cargo.toml
 
 # Copy binary to output directory
 echo "Copying binary to output directory..."
 mkdir -p "../../$OUTPUT_DIR"
-cp target/release/reth "../../$OUTPUT_DIR/"
+cp target/maxperf/op-reth "../../$OUTPUT_DIR/"
 
 echo "reth binary built successfully and placed in $OUTPUT_DIR/reth" 

clients/versions.env

@@ -4,7 +4,7 @@
 
 # Reth Configuration
 RETH_REPO="https://github.com/paradigmxyz/reth/"
-RETH_VERSION="v1.5.1"
+RETH_VERSION="v1.8.1"
 
 # Op-Geth Configuration  
 GETH_REPO="https://github.com/ethereum-optimism/op-geth/"

configs/examples/snapshot.yml

@@ -1,44 +1,38 @@
-name: Snapshot Functionality Test
+name: Two-Tier Snapshot Functionality Test
 description: |
-  Snapshot Functionality Test - Tests client snapshot creation and loading capabilities to validate state snapshot performance for fast sync operations.
+  Two-Tier Snapshot Functionality Test - Demonstrates the new optimized snapshot system with initial snapshots and per-test copying.
 
-  This benchmark suite tests snapshot functionality with both Sepolia Alpha and development network data, including snapshot creation, loading, and validation processes. Features skip-if-nonempty optimization for development efficiency and tests multiple gas limit configurations.
+  This benchmark suite uses a two-tier snapshot approach:
+  1. Initial snapshots are downloaded once at benchmark startup and stored persistently
+  2. Per-test snapshots are copied from initial snapshots for each test run using rsync
+  3. Test-specific copies are cleaned up after each test while preserving initial snapshots
 
-  Use Case: Validate state snapshot performance for fast sync operations, test snapshot creation and loading capabilities across different environments, and ensure snapshot performance remains consistent in development workflows.
+  Use Case: Optimized snapshot performance for fast test execution, reduced network overhead, and efficient storage management across multiple node types.
 
 payloads:
   - name: Transfer-only
     id: transfer-only
     type: transfer-only
 
 benchmarks:
-  - snapshot:
-      # skip non-empty for testing so we don't copy every time we run this
-      # just delete the snapshot directory to force a full copy
-      command: ./scripts/setup-snapshot.sh --skip-if-nonempty
-      genesis_file: ../../sepolia-alpha/sepolia-alpha-genesis.json
-    # force_clean is true by default to ensure consistency, but we can skip it for testing
-    force_clean: false
+  - initial_snapshots:
+      - node_type: reth
+        # Download an initial reth snapshot that can be copied for each test
+        command: ./scripts/setup-initial-snapshot.sh --network=sepolia --node-type=reth --destination=/data/snapshots/reth/initial
+        destination: /data/snapshots/reth/initial
+        superchain_chain_id: 84532
+      - node_type: geth  
+        # Download an initial geth snapshot that can be copied for each test
+        command: ./scripts/setup-initial-snapshot.sh --network=sepolia --node-type=geth --destination=/data/snapshots/geth/initial
+        destination: /data/snapshots/geth/initial
+        superchain_chain_id: 84532
     variables:
       - type: payload
         value: transfer-only
       - type: node_type
         values:
           - reth
-      - type: num_blocks
-        value: 10
-      - type: gas_limit
-        values:
-          - 15000000
-          - 30000000
-          - 60000000
-          - 90000000
-  - variables:
-      - type: payload
-        value: transfer-only
-      - type: node_type
-        values:
-          - reth
+          - geth
       - type: num_blocks
         value: 10
       - type: gas_limit

configs/public/basic.yml

@@ -23,7 +23,7 @@ benchmarks:
         value: 10
       - type: gas_limit
         values:
-          - 15000000
+          - 20000000
           - 30000000
           - 60000000
           - 90000000

go.mod

@@ -5,12 +5,14 @@ go 1.23.0
 toolchain go1.23.10
 
 require (
+	github.com/aws/aws-sdk-go v1.47.13
 	github.com/charmbracelet/huh v0.7.0
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc
 	github.com/ethereum-optimism/optimism v1.13.3
 	github.com/ethereum/go-ethereum v1.16.0
 	github.com/holiman/uint256 v1.3.2
 	github.com/pkg/errors v0.9.1
+	github.com/prometheus/client_model v0.6.2
 	github.com/prometheus/common v0.62.0
 	github.com/stretchr/testify v1.10.0
 	github.com/urfave/cli/v2 v2.27.6
@@ -35,15 +37,13 @@ require (
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/charmbracelet/bubbles v0.21.0 // indirect
 	github.com/charmbracelet/bubbletea v1.3.4 // indirect
-	github.com/charmbracelet/bubbletea/v2 v2.0.0-beta1 // indirect
 	github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc // indirect
 	github.com/charmbracelet/lipgloss v1.1.0 // indirect
 	github.com/charmbracelet/x/ansi v0.8.0 // indirect
 	github.com/charmbracelet/x/cellbuf v0.0.13 // indirect
+	github.com/charmbracelet/x/exp/golden v0.0.0-20241212170349-ad4b7ae0f25f // indirect
 	github.com/charmbracelet/x/exp/strings v0.0.0-20240722160745-212f7b056ed0 // indirect
-	github.com/charmbracelet/x/input v0.3.4 // indirect
 	github.com/charmbracelet/x/term v0.2.1 // indirect
-	github.com/charmbracelet/x/windows v0.2.0 // indirect
 	github.com/cockroachdb/errors v1.11.3 // indirect
 	github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce // indirect
 	github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect
@@ -82,6 +82,7 @@ require (
 	github.com/ipfs/go-datastore v0.6.0 // indirect
 	github.com/jackpal/go-nat-pmp v1.0.2 // indirect
 	github.com/jbenet/goprocess v0.1.4 // indirect
+	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/klauspost/compress v1.18.0 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.9 // indirect
 	github.com/kr/pretty v0.3.1 // indirect
@@ -121,7 +122,6 @@ require (
 	github.com/pion/transport/v3 v3.0.7 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/prometheus/client_golang v1.22.0 // indirect
-	github.com/prometheus/client_model v0.6.2 // indirect
 	github.com/prometheus/procfs v0.15.1 // indirect
 	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/rogpeppe/go-internal v1.13.1 // indirect