diff --git a/.github/workflows/build-and-push.yml b/.github/workflows/build-and-push.yml new file mode 100644 index 0000000..f813c3c --- /dev/null +++ b/.github/workflows/build-and-push.yml @@ -0,0 +1,113 @@ +name: Release and Publish Docker Images + +on: + workflow_dispatch: + inputs: + version: + description: 'Release version (e.g. v1.2.3)' + required: true + type: string + +env: + REGISTRY: ghcr.io + IMAGE_PREFIX: ${{ github.repository }} + VERSION: ${{ github.event.inputs.version }} + +jobs: + build-binaries: + runs-on: ubuntu-latest + strategy: + matrix: + include: + - goos: linux + goarch: amd64 + - goos: linux + goarch: arm64 + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: '1.24' + + - name: Build test-runner + env: + GOOS: ${{ matrix.goos }} + GOARCH: ${{ matrix.goarch }} + CGO_ENABLED: 0 + run: | + go build -o test-runner-${{ matrix.goos }}-${{ matrix.goarch }} ./cmd/test-runner + + - name: Build snapshot-checker + env: + GOOS: ${{ matrix.goos }} + GOARCH: ${{ matrix.goarch }} + CGO_ENABLED: 0 + run: | + go build -o snapshot-checker-${{ matrix.goos }}-${{ matrix.goarch }} ./cmd/snapshot-checker + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: binaries-${{ matrix.goos }}-${{ matrix.goarch }} + path: | + test-runner-${{ matrix.goos }}-${{ matrix.goarch }} + snapshot-checker-${{ matrix.goos }}-${{ matrix.goarch }} + + docker: + needs: build-binaries + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + strategy: + matrix: + include: + - name: test-runner + dockerfile: Dockerfile.test-runner + - name: snapshot-checker + dockerfile: Dockerfile.snapshot-checker + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to Container Registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push Docker image + uses: docker/build-push-action@v5 + with: + context: . + file: ${{ matrix.dockerfile }} + push: true + tags: | + ${{ env.REGISTRY }}/${{ env.IMAGE_PREFIX }}/${{ matrix.name }}:${{ env.VERSION }} + ${{ env.REGISTRY }}/${{ env.IMAGE_PREFIX }}/${{ matrix.name }}:latest + cache-from: type=gha + cache-to: type=gha,mode=max + + release: + needs: [build-binaries, docker] + runs-on: ubuntu-latest + steps: + - name: Download all artifacts + uses: actions/download-artifact@v4 + with: + path: binaries + merge-multiple: true + + - name: Create Release + uses: softprops/action-gh-release@v1 + with: + tag_name: ${{ env.VERSION }} + files: binaries/* + generate_release_notes: true diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 6c07cee..0000000 --- a/Dockerfile +++ /dev/null @@ -1,41 +0,0 @@ -# ===== STAGE 1: Build Stage ===== -FROM golang:1.24 AS builder - -# Set environment variables -ENV CGO_ENABLED=1 \ - GOOS=linux \ - GOARCH=amd64 - -# Install necessary build dependencies -RUN apt-get update && apt-get install -y gcc libc-dev - -# Set working directory inside the container -WORKDIR /app - -# Copy Go modules manifests first (for better caching) -COPY go.mod go.sum ./ -RUN go mod download - -# Copy the entire source code -COPY . . - -# Build the Go binary -RUN go build -o /app/server ./cmd/main.go - -# ===== STAGE 2: Runtime Stage ===== -FROM debian:stable-slim - -# Install required runtime dependencies (for CGO) -RUN apt-get update && apt-get install -y ca-certificates libc6 && rm -rf /var/lib/apt/lists/* - -# Set working directory -WORKDIR /app - -# Copy the compiled binary from builder stage -COPY --from=builder /app/server . - -# Expose the application port -EXPOSE 8080 - -# Run the application -CMD ["./server"] diff --git a/Dockerfile.snapshot-checker b/Dockerfile.snapshot-checker new file mode 100644 index 0000000..0f5ee66 --- /dev/null +++ b/Dockerfile.snapshot-checker @@ -0,0 +1,30 @@ +# Build stage +FROM golang:1.24-alpine AS builder + +WORKDIR /app + +# Install git for fetching dependencies +RUN apk add --no-cache git + +# Copy go mod files first for better caching +COPY go.mod go.sum ./ +RUN go mod download + +# Copy source code +COPY . . + +# Build the snapshot-checker application +RUN CGO_ENABLED=0 GOOS=linux go build -o /app/snapshot-checker ./cmd/snapshot-checker + +# Final stage +FROM alpine:latest + +WORKDIR /app + +# Install ca-certificates for HTTPS requests and docker-cli for snapshot operations +RUN apk add --no-cache ca-certificates docker-cli + +# Copy binary from builder +COPY --from=builder /app/snapshot-checker /app/snapshot-checker + +ENTRYPOINT ["/app/snapshot-checker"] diff --git a/Dockerfile.test-runner b/Dockerfile.test-runner new file mode 100644 index 0000000..1073785 --- /dev/null +++ b/Dockerfile.test-runner @@ -0,0 +1,30 @@ +# Build stage +FROM golang:1.24-alpine AS builder + +WORKDIR /app + +# Install git for fetching dependencies +RUN apk add --no-cache git + +# Copy go mod files first for better caching +COPY go.mod go.sum ./ +RUN go mod download + +# Copy source code +COPY . . + +# Build the test-runner application +RUN CGO_ENABLED=0 GOOS=linux go build -o /app/test-runner ./cmd/test-runner + +# Final stage +FROM alpine:latest + +WORKDIR /app + +# Install ca-certificates for HTTPS requests and docker-cli for snapshot operations +RUN apk add --no-cache ca-certificates docker-cli + +# Copy binary from builder +COPY --from=builder /app/test-runner /app/test-runner + +ENTRYPOINT ["/app/test-runner"] diff --git a/README.md b/README.md index 650afc2..cdf8d6e 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,235 @@ # Dappnode ethereum clients test SDK +A testing utility for Dappnode staker packages that runs in GitHub self-hosted runners. It can execute tests either manually or automatically on GitHub pull requests, with automatic PR commenting for test reports. + +## Features + +- **Automated Testing**: Tests staker packages (execution, consensus, web3signer, etc.) +- **GitHub Integration**: Automatically comments test reports on pull requests +- **Timing Measurements**: Measures duration of all test phases (setup, execution, cleanup) +- **Container Log Collection**: Captures error logs from all relevant containers during test execution +- **Detailed Reports**: Generates comprehensive reports with clients used, timings, and error logs +- **Snapshot Management**: Automatically downloads and manages execution client snapshots + +## Quick Start (CI Usage) + +### Using Docker Images from GHCR + +Docker images are automatically published to GitHub Container Registry on every push to main and on releases. + +```bash +# Test Runner +ghcr.io/dappnode/staker-test-util/test-runner:latest + +# Snapshot Checker +ghcr.io/dappnode/staker-test-util/snapshot-checker:latest +``` + +### Test Runner in GitHub Actions + +```yaml +- name: Run Staker Tests + run: | + docker run --rm \ + --network dncore_network \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /var/lib/docker/volumes:/var/lib/docker/volumes \ + -e IPFS_GATEWAY_URL=http://ipfs.dappnode:8080 \ + -e IPFS_HASH=${{ env.IPFS_HASH }} \ + -e EXECUTION_CLIENT=geth \ + -e CONSENSUS_CLIENT=nimbus \ + -e GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} \ + -e GITHUB_REPOSITORY=${{ github.repository }} \ + -e GITHUB_PR_NUMBER=${{ github.event.pull_request.number }} \ + ghcr.io/dappnode/staker-test-util/test-runner:latest +``` + +### Snapshot Checker in GitHub Actions + +```yaml +- name: Ensure Snapshot Available + run: | + docker run --rm \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /var/lib/docker/volumes:/var/lib/docker/volumes \ + -e EXECUTION_CLIENT=geth \ + -e NETWORK=hoodi \ + -e RUN_ONCE=true \ + ghcr.io/dappnode/staker-test-util/snapshot-checker:latest +``` + +See [examples/workflows/](examples/workflows/) for complete workflow examples. + +## Configuration + +### Test Runner Environment Variables + +| Variable | Description | Required | +|----------|-------------|----------| +| `IPFS_GATEWAY_URL` | IPFS gateway URL for fetching packages | Yes | +| `IPFS_HASH` | IPFS hash of the test package | Yes | +| `EXECUTION_CLIENT` | Override execution client (geth, reth, nethermind, besu, erigon) | No | +| `CONSENSUS_CLIENT` | Override consensus client (prysm, teku, nimbus, lodestar) | No | +| `LOG_LEVEL` | Log level (DEBUG, INFO, WARN, ERROR) | No | + +> **Note:** If the package being tested is an execution or consensus client, it will override the respective environment variable with a warning. + +### Snapshot Checker Environment Variables + +| Variable | Description | Required | +|----------|-------------|----------| +| `EXECUTION_CLIENT` | Execution client to manage (geth, reth, nethermind, besu, erigon) | Yes | +| `NETWORK` | Network name (default: hoodi) | No | +| `CRON_INTERVAL_SEC` | Interval between checks in seconds (default: 21600 = 6h) | No | +| `RUN_ONCE` | Run once and exit (default: false) | No | +| `LOG_LEVEL` | Log level (DEBUG, INFO, WARN, ERROR) | No | + +### GitHub Integration (Optional) + +These environment variables enable automatic PR commenting. When running in GitHub Actions, most of these are set automatically. + +| Variable | Description | GitHub Actions Auto-set | +|----------|-------------|------------------------| +| `GITHUB_TOKEN` | GitHub token with PR comment permissions | ✅ (via `${{ secrets.GITHUB_TOKEN }}`) | +| `GITHUB_REPOSITORY` | Repository in `owner/repo` format | ✅ | +| `GITHUB_PR_NUMBER` | Pull request number | ❌ (extract from event) | +| `GITHUB_RUN_ID` | GitHub Actions run ID (for linking to logs) | ✅ | +| `GITHUB_SERVER_URL` | GitHub server URL (default: `https://github.com`) | ✅ | + +### CLI Flags + +All environment variables can also be set via CLI flags: + +```bash +# Test Runner +./test-runner \ + --ipfs-gateway-url="http://ipfs.dappnode:8080" \ + --ipfs-hash="QmSfPFSauovbMzEcvf2a2csoHtfqpViShwEYpuX3fPR8zv" \ + --execution-client="geth" \ + --consensus-client="nimbus" \ + --github-token="ghp_xxxx" \ + --github-repository="dappnode/staker-test-util" \ + --github-pr-number="123" + +# Snapshot Checker +./snapshot-checker \ + --execution-client="geth" \ + --cron-interval=21600 \ + --run-once +``` + +## GitHub Actions Workflow Example + +```yaml +name: Staker Test + +on: + pull_request: + types: [opened, synchronize, reopened] + +jobs: + test: + runs-on: self-hosted + steps: + - name: Run Staker Test + env: + IPFS_GATEWAY_URL: http://ipfs.dappnode:8080 + IPFS_HASH: ${{ github.event.inputs.ipfs_hash }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_PR_NUMBER: ${{ github.event.pull_request.number }} + GITHUB_RUN_ID: ${{ github.run_id }} + GITHUB_SERVER_URL: ${{ github.server_url }} + run: | + docker-compose up --build +``` + +## Test Report + +The test report includes: + +### Clients Used +- Execution client DNP name +- Consensus client DNP name +- Web3Signer DNP name +- MEV Boost DNP name +- Network + +### Timing Measurements + +#### Environment Setup +- SetStakerConfig +- PackageInstall +- StopAndGetVolumeTarget +- DownloadAndExtractSnapshot +- StartContainer + +#### Test Execution +- WaitForBeaconchainSync +- WaitForExecutionSync +- WaitForValidatorLiveness + +### Container Error Logs +Captures error lines from: +- Brain container +- Signer container +- Beaconchain container +- Validator container +- Execution container + +> Shows up to 3 error lines per container. Full logs available in CI. + +## Docker Compose + +```bash +# With GitHub integration +GITHUB_TOKEN=ghp_xxxx \ +GITHUB_REPOSITORY=dappnode/staker-test-util \ +GITHUB_PR_NUMBER=123 \ +docker-compose up --build +``` + ## TODO's -- [ ] Collect logs from containers and create a report -- [ ] Add to the report the clients used -- [x] Add to ensurer wait for consensus sync. + +- snapshot checker (cron that runs on startup and every 6h): ensures the Execution clients have the latest snapshots downloaded and mounted to their respective volumes + - Checks if snapshots exists for given clients and network + - If they dont exist then write temporary file `download_in_progress` to signal other processes that snapshot download is in progress and: + 1. download snapshots + 2. extract snapshots + 3. mount snapshots to respective volumes + 4. Write the block number of the snapshot downloaded to a file `snapshot_block_number` inside the respective volume + 5. remove temporary file `download_in_progress` + - If they do exist then check: + 1. If file `download_in_progress` exists then exit + 2. If file `snapshot_block_number` exists then check if the latest snapshot available is newer, if so redownload snapshot as above + 3. If file `snapshot_block_number` does not exist then redownload snapshot as above +- test runner: executed once per test (manually or on PR) + - ensures environment before running tests + - among other things it must check if the file `download_in_progress` exits, if so the test must wait until it is removed + - runs tests + - cleanup environment after tests + +- [x] Implement a github adapter to interact with issues and PRs so we can automate report creation as well as the testing process. +- [x] Measure the time it takes every process in the test and add it to the report +- [x] Collect logs from containers and create a report +- [x] Add to the report the clients used +- [x] Implement timer check in the snapshot checker +- [ ] The test runner must remove the clients or unset the staker config on exit +- [x] Implement a composite adapter for the snapshot checker +- [x] Implement time tracker in the snapshot checker service to measure time taken for each action, specially download and extraction of snapshots +- [ ] Auto-updates for this dappnode must run much more often than production, so clients are always updated to latest versions +- [ ] Research how to release this SDK tool to be run from a github action directly +- [ ] Implement when manual trigger (`workflow_dispatch`) the clients will be passed as inputs, use them to create the staker config for the test +- [ ] Implement edit of `/usr/src/dappnode/DNCORE/docker-compose.yml` file to add env `TEST=true` and relaunch compose +- [ ] Print version of the clients + - [ ] For the EC it must be printed +- [ ] Add to the report the block of the snapshot used +- [ ] Consider always removing beacon volumes to ensure avoiding old states of chain and always start with the checkpoint sync +- [ ] Implement switch off of dappmanager cron that restarts containers of clients selected in stakers +- [ ] Silent the tar output when extracting snapshots or make it less verbose +- [ ] Consider adding a DB for reports and a UI to consume them - [ ] Consider adding to report beaconcha validator url -- [x] Add a handle signal to stop the test gracefully: release mount data if any, remove packages if any installed. -> run cleaner -- [ ] Consider setting keystore and password through github secrets. \ No newline at end of file +- [ ] Consider adding support for multiple networks (mainnet, prater, etc.) +- [ ] Consider setting keystore and password through github secrets. +- [ ] How to ensure IPFS resilience, it must resolve production and dev IPFS hashes! +- [ ] Pass all the time measured in the ensurer and or runner and or cleaner to the test runner service for printing and report \ No newline at end of file diff --git a/cmd/snapshot-checker/main.go b/cmd/snapshot-checker/main.go new file mode 100644 index 0000000..a4ef6c9 --- /dev/null +++ b/cmd/snapshot-checker/main.go @@ -0,0 +1,130 @@ +package main + +import ( + "clients-test/internal/adapters/apis/docker" + "clients-test/internal/adapters/apis/snapshots" + "clients-test/internal/adapters/composite/snapshotmanager" + "clients-test/internal/adapters/shared/blocknumber" + "clients-test/internal/adapters/shared/download" + "clients-test/internal/adapters/shared/testing" + "clients-test/internal/application/domain" + "clients-test/internal/application/services" + "clients-test/internal/config" + "clients-test/internal/logger" + "context" + "fmt" + "os" + "os/signal" + "strings" + "syscall" +) + +var logPrefix = "SNAPSHOT_CHECKER" + +func main() { + logger.InfoWithPrefix(logPrefix, "Starting Snapshot Checker service") + + // Parse and validate configuration + cfg := config.ParseSnapshotCheckerConfig() + cfg.Validate() + + // Get execution client info + executionClient, ok := domain.GetExecutionClient(cfg.Network, cfg.ExecutionClient) + if !ok { + logger.FatalWithPrefix(logPrefix, "Failed to get execution client info for '%s'", cfg.ExecutionClient) + } + + // Initialize domain config + snapshotConfig := domain.SnapshotCheckerConfig{ + ExecutionClient: executionClient, + CronIntervalSec: cfg.CronIntervalSec, + Network: cfg.Network, + } + + // Print configuration + printSnapshotCheckerConfig(logPrefix, snapshotConfig) + + // Set up Ctrl+C (SIGINT) handler + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + sigs := make(chan os.Signal, 1) + signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM) + + // Initialize adapters + snapshotsAdapter := snapshots.NewSnapshotsAdapter() + dockerAdapter, err := docker.NewDockerAdapter() + if err != nil { + logger.FatalWithPrefix(logPrefix, "Failed to init DockerAdapter: %v", err) + } + + // Initialize progress adapters with client's volume path + logger.InfoWithPrefix(logPrefix, "Using volume path for flag files: %s", executionClient.VolumeTargetPath) + downloadAdapter := download.NewDownloadAdapterWithPath(executionClient.VolumeTargetPath) + testAdapter := testing.NewTestAdapterWithPath(executionClient.VolumeTargetPath) + blockNumberAdapter := blocknumber.NewBlockNumberAdapterWithPath(executionClient.VolumeTargetPath) + + // Create composite snapshot manager adapter + snapshotManagerAdapter := snapshotmanager.NewSnapshotManagerAdapter( + snapshotsAdapter, + dockerAdapter, + ) + + // Clear any stale download marker on startup + if inProgress, err := downloadAdapter.IsDownloadInProgress(ctx); err != nil { + logger.WarnWithPrefix(logPrefix, "[%s] Failed to check %s marker: %v", executionClient.ShortName, domain.ProgressFileName, err) + } else if inProgress { + logger.WarnWithPrefix(logPrefix, "[%s] Found stale %s marker; clearing it on startup", executionClient.ShortName, domain.ProgressFileName) + if err := downloadAdapter.ClearDownloadInProgress(context.Background()); err != nil { + logger.WarnWithPrefix(logPrefix, "[%s] Failed to clear %s marker on startup: %v", executionClient.ShortName, domain.ProgressFileName, err) + } + } + + // Initialize the snapshot checker service + service := services.NewSnapshotCheckerService( + snapshotManagerAdapter, + downloadAdapter, + testAdapter, + blockNumberAdapter, + snapshotConfig, + ) + + go func() { + sig := <-sigs + logger.InfoWithPrefix(logPrefix, "Received signal: %v, shutting down...", sig) + // Stop any running download container using the service method + service.StopDownload(context.Background()) + // Best-effort cleanup: clear marker file so next run isn't blocked. + service.ClearDownloadMarker(context.Background()) + cancel() + }() + + if err := service.Start(ctx, cfg.RunOnce); err != nil { + if err == context.Canceled { + logger.InfoWithPrefix(logPrefix, "Snapshot checker stopped gracefully") + } else { + logger.FatalWithPrefix(logPrefix, "Snapshot checker failed: %v", err) + } + } + + logger.InfoWithPrefix(logPrefix, "Snapshot checker service stopped") +} + +// helper to pretty print snapshot checker config +func printSnapshotCheckerConfig(prefix string, sc domain.SnapshotCheckerConfig) { + c := sc.ExecutionClient + + var b strings.Builder + b.WriteString("SnapshotCheckerConfig:\n") + b.WriteString(fmt.Sprintf(" Network: %s\n", sc.Network)) + b.WriteString(fmt.Sprintf(" CronIntervalSec: %d\n", sc.CronIntervalSec)) + b.WriteString(" ExecutionClient:\n") + b.WriteString(fmt.Sprintf(" ShortName: %s\n", c.ShortName)) + b.WriteString(fmt.Sprintf(" DnpName: %s\n", c.DnpName)) + b.WriteString(fmt.Sprintf(" VolumeName: %s\n", c.VolumeName)) + b.WriteString(fmt.Sprintf(" ContainerName: %s\n", c.ContainerName)) + b.WriteString(fmt.Sprintf(" VolumeTargetPath: %s\n", c.VolumeTargetPath)) + + msg := b.String() + logger.InfoWithPrefix(prefix, "%s", msg) +} diff --git a/cmd/main.go b/cmd/test-runner/main.go similarity index 50% rename from cmd/main.go rename to cmd/test-runner/main.go index 4fbcbe6..43f6631 100644 --- a/cmd/main.go +++ b/cmd/test-runner/main.go @@ -6,65 +6,64 @@ import ( "clients-test/internal/adapters/apis/dappmanager" "clients-test/internal/adapters/apis/docker" "clients-test/internal/adapters/apis/execution" + "clients-test/internal/adapters/apis/github" "clients-test/internal/adapters/apis/ipfs" - "clients-test/internal/adapters/apis/tropidatooor" + "clients-test/internal/adapters/apis/snapshots" "clients-test/internal/adapters/composite" - "clients-test/internal/adapters/system/mount" + "clients-test/internal/adapters/shared/blocknumber" + "clients-test/internal/adapters/shared/download" + "clients-test/internal/adapters/shared/testing" "clients-test/internal/application/domain" "clients-test/internal/application/services" + "clients-test/internal/config" "clients-test/internal/logger" "context" - "flag" "fmt" "os" "os/signal" "syscall" ) -var logPrefix = "MAIN" +var logPrefix = "TEST_RUNNER" func main() { - logger.InfoWithPrefix(logPrefix, "Starting Notifications service") + logger.InfoWithPrefix(logPrefix, "Starting Test Runner service") - // Set up Ctrl+C (SIGINT) handler to call composite cleaner - cleanupDone := make(chan struct{}) - sigs := make(chan os.Signal, 1) - signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM) + // Parse and validate configuration + cfg := config.ParseConfig() + cfg.Validate() - // CLI flags - ipfsGatewayUrl := flag.String("ipfs-gateway-url", "", "IPFS gateway URL (required)") - tropidatooorUrl := flag.String("tropidatooor-url", "", "Tropidatooor API URL (required)") - ipfsHash := flag.String("ipfs-hash", "", "IPFS hash for the test package (required)") - flag.Parse() + // Set up Ctrl+C (SIGINT) handler + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() - if *ipfsGatewayUrl == "" || *tropidatooorUrl == "" || *ipfsHash == "" { - logger.FatalWithPrefix(logPrefix, "All flags --ipfs-gateway-url, --tropidatooor-url, and --ipfs-hash are required.") - } - - ctx := context.Background() + sigs := make(chan os.Signal, 1) + signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM) // Fetch dnpName from ipfs hash - ipfsAdapter := ipfs.NewIPFSAdapter(ipfsGatewayUrl) - pkg, err := ipfsAdapter.GetDnpNameAndServiceName(ctx, *ipfsHash) + ipfsAdapter := ipfs.NewIPFSAdapter(&cfg.IPFSGatewayURL) + pkg, err := ipfsAdapter.GetDnpNameAndServiceName(ctx, cfg.IPFSHash) if err != nil { logger.FatalWithPrefix(logPrefix, "Failed to get dnpName from IPFS hash: %v", err) } - // Retrieve staker config based on pkg (dnpName and serviceName) - stakerConfig := domain.StakerConfigForNetwork(pkg) + // Retrieve staker config based on pkg (dnpName and serviceName) with optional overrides + overrides := domain.ClientOverrides{ + ExecutionClient: cfg.ExecutionClient, + ConsensusClient: cfg.ConsensusClient, + } + stakerConfig, warnings := domain.StakerConfigForNetwork(pkg, overrides) + + // Log any warnings from client resolution + for _, warning := range warnings { + logger.WarnWithPrefix(logPrefix, "%s", warning) + } // print the staker config for debugging with each item on a new line printStakerConfig(logPrefix, stakerConfig) - // Get mount path - tropidatooorAdapter := tropidatooor.NewTropidatooorAdapter(*tropidatooorUrl) - mountConfig, err := tropidatooorAdapter.DataRequest(ctx, stakerConfig.DataBackendName) - if err != nil { - logger.FatalWithPrefix(logPrefix, "Failed to get mount path: %v", err) - } - // Initialize API adapters - mountAdapter := mount.NewMountAdapter() + snapshotsAdapter := snapshots.NewSnapshotsAdapter() dappManagerAdapter := dappmanager.NewDappManagerAdapter() brainAdapter := brain.NewBrainAdapter(stakerConfig.Urls.BrainURL) beaconchainAdapter := beaconchain.NewBeaconchainAdapter(stakerConfig.Urls.BeaconchainURL) @@ -74,36 +73,54 @@ func main() { logger.FatalWithPrefix(logPrefix, "Failed to init DockerAdapter: %v", err) } + // Initialize GitHub adapter for PR commenting + githubAdapter := github.NewGitHubAdapter(cfg.GitHub) + + // Initialize shared adapters with execution client's volume path + logger.InfoWithPrefix(logPrefix, "Using volume path for flag files: %s", stakerConfig.ExecutionVolumeTargetPath) + downloadAdapter := download.NewDownloadAdapterWithPath(stakerConfig.ExecutionVolumeTargetPath) + testAdapter := testing.NewTestAdapterWithPath(stakerConfig.ExecutionVolumeTargetPath) + blockAdapter := blocknumber.NewBlockNumberAdapterWithPath(stakerConfig.ExecutionVolumeTargetPath) + + // Log GitHub configuration status + if githubAdapter.IsEnabled() { + logger.InfoWithPrefix(logPrefix, "GitHub integration enabled - will comment on PR #%d", cfg.GitHub.PRNumber) + } else { + logger.InfoWithPrefix(logPrefix, "GitHub integration not enabled (missing token, repository, or PR number)") + } + // Initialize the unified test adapter (now also initializes composites internally) compositeAdapter := composite.NewCompositeAdapter( dappManagerAdapter, brainAdapter, - tropidatooorAdapter, dockerAdapter, - mountAdapter, + snapshotsAdapter, beaconchainAdapter, executionAdapter, ipfsAdapter, + githubAdapter, + blockAdapter, ) // Ctrl+C handler: call CleanEnvironment on composite go func() { sig := <-sigs - logger.InfoWithPrefix(logPrefix, "Received signal: %v, running cleanup...", sig) - err := compositeAdapter.CleanEnvironment(ctx, stakerConfig, *mountConfig) + logger.InfoWithPrefix(logPrefix, "Received signal: %v, shutting down...", sig) + err := compositeAdapter.CleanEnvironment(context.Background(), stakerConfig) if err != nil { logger.ErrorWithPrefix(logPrefix, "Cleanup failed: %v", err) - } else { - logger.InfoWithPrefix(logPrefix, "Cleanup completed successfully") } - close(cleanupDone) - os.Exit(1) + // Best-effort cleanup, clear marker file `.test_in_progress` so next run isn't blocked. + if err := testAdapter.ClearTestInProgress(context.Background()); err != nil { + logger.WarnWithPrefix(logPrefix, "Failed to clear %s marker on shutdown: %v", domain.TestProgressFileName, err) + } + cancel() }() // Initialize and run the service - testRunner := services.NewTestRunner(compositeAdapter) + testRunner := services.NewTestRunner(compositeAdapter, downloadAdapter, testAdapter) - if err := testRunner.RunTest(ctx, *mountConfig, stakerConfig, pkg); err != nil { + if err := testRunner.RunTest(ctx, stakerConfig, pkg); err != nil { logger.FatalWithPrefix(logPrefix, "Test run failed: %v", err) } @@ -120,7 +137,7 @@ func printStakerConfig(prefix string, sc domain.StakerConfig) { MevBoostDnpName: %s Network: %s ExecutionContainerName: %s - DataBackendName: %s + ExecutionVolumeTargetPath: %s Urls: ExecutionURL: %s BrainURL: %s @@ -133,7 +150,7 @@ func printStakerConfig(prefix string, sc domain.StakerConfig) { sc.MevBoostDnpName, sc.Network, sc.ExecutionContainerName, - sc.DataBackendName, + sc.ExecutionVolumeTargetPath, sc.Urls.ExecutionURL, sc.Urls.BrainURL, sc.Urls.BeaconchainURL, diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml new file mode 100644 index 0000000..d938996 --- /dev/null +++ b/docker-compose-dev.yml @@ -0,0 +1,45 @@ +services: + snapshot-checker: + build: + context: . + dockerfile: Dockerfile.snapshot-checker + container_name: snapshot-checker + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - /var/lib/docker/volumes:/var/lib/docker/volumes + environment: + - LOG_LEVEL=DEBUG + - EXECUTION_CLIENT=${EXECUTION_CLIENT:-geth} + - NETWORK=${NETWORK:-hoodi} + - CRON_INTERVAL_SEC=${CRON_INTERVAL_SEC:-21600} + - RUN_ONCE=${RUN_ONCE:-true} + networks: + - dncore_network + + test-runner: + build: + context: . + dockerfile: Dockerfile.test-runner + container_name: test-runner + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - /var/lib/docker/volumes:/var/lib/docker/volumes + environment: + - LOG_LEVEL=DEBUG + - IPFS_GATEWAY_URL=http://ipfs.dappnode:8080 + - IPFS_HASH=QmSfPFSauovbMzEcvf2a2csoHtfqpViShwEYpuX3fPR8zv + - EXECUTION_CLIENT=${EXECUTION_CLIENT:-geth} + - CONSENSUS_CLIENT=${CONSENSUS_CLIENT:-nimbus} + # GitHub integration (optional - for PR commenting) + - GITHUB_TOKEN=${GITHUB_TOKEN:-} + - GITHUB_REPOSITORY=${GITHUB_REPOSITORY:-} + - GITHUB_PR_NUMBER=${GITHUB_PR_NUMBER:-} + - GITHUB_RUN_ID=${GITHUB_RUN_ID:-} + - GITHUB_SERVER_URL=${GITHUB_SERVER_URL:-https://github.com} + # command: ["--ipfs-gateway-url", "${IPFS_GATEWAY_URL}", "--ipfs-hash", "${IPFS_HASH}"] + networks: + - dncore_network + +networks: + dncore_network: + external: true diff --git a/examples/workflows/complete-staker-ci.yml b/examples/workflows/complete-staker-ci.yml new file mode 100644 index 0000000..e187e88 --- /dev/null +++ b/examples/workflows/complete-staker-ci.yml @@ -0,0 +1,75 @@ +# Example: Complete CI workflow for DAppNode staker packages +# +# This workflow: +# 1. Builds your package +# 2. Publishes to IPFS +# 3. Ensures snapshot is available +# 4. Runs staker tests +# +# Copy and adapt for your package repository. + +name: Complete Staker CI + +on: + pull_request: + branches: + - main + push: + branches: + - main + +jobs: + build: + runs-on: ubuntu-latest + outputs: + ipfs_hash: ${{ steps.publish.outputs.ipfs_hash }} + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Build DAppNode Package + run: | + # Your build steps here + # npx @dappnode/dappnodesdk build + echo "Building package..." + + - name: Publish to IPFS + id: publish + run: | + # Your IPFS publish steps here + # This should output the IPFS hash + echo "ipfs_hash=QmYourIPFSHash" >> $GITHUB_OUTPUT + + ensure-snapshot: + runs-on: self-hosted + needs: build + steps: + - name: Ensure Snapshot Available + run: | + docker run --rm \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /var/lib/docker/volumes:/var/lib/docker/volumes \ + -e EXECUTION_CLIENT=geth \ + -e NETWORK=hoodi \ + -e RUN_ONCE=true \ + ghcr.io/dappnode/staker-test-util/snapshot-checker:latest + + test: + runs-on: self-hosted + needs: [build, ensure-snapshot] + steps: + - name: Run Staker Tests + run: | + docker run --rm \ + --network dncore_network \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /var/lib/docker/volumes:/var/lib/docker/volumes \ + -e IPFS_GATEWAY_URL=http://ipfs.dappnode:8080 \ + -e IPFS_HASH=${{ needs.build.outputs.ipfs_hash }} \ + -e EXECUTION_CLIENT=geth \ + -e CONSENSUS_CLIENT=nimbus \ + -e GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} \ + -e GITHUB_REPOSITORY=${{ github.repository }} \ + -e GITHUB_PR_NUMBER=${{ github.event.pull_request.number }} \ + -e GITHUB_RUN_ID=${{ github.run_id }} \ + ghcr.io/dappnode/staker-test-util/test-runner:latest diff --git a/examples/workflows/snapshot-checker.yml b/examples/workflows/snapshot-checker.yml new file mode 100644 index 0000000..399fb0f --- /dev/null +++ b/examples/workflows/snapshot-checker.yml @@ -0,0 +1,51 @@ +# Example: Running snapshot-checker as a scheduled job +# +# This workflow runs the snapshot-checker to keep execution client snapshots updated. +# Can run as Docker container or as a binary. +# +# Prerequisites: +# - Self-hosted runner with access to Docker volumes + +name: Snapshot Checker + +on: + schedule: + # Run every 6 hours + - cron: '0 */6 * * *' + workflow_dispatch: + inputs: + execution_client: + description: 'Execution client (geth, reth, nethermind, besu, erigon)' + required: true + default: 'geth' + +jobs: + # Option 1: Run as Docker container + snapshot-check-docker: + runs-on: self-hosted + steps: + - name: Run Snapshot Checker (Docker) + run: | + docker run --rm \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /var/lib/docker/volumes:/var/lib/docker/volumes \ + -e EXECUTION_CLIENT=${{ github.event.inputs.execution_client || 'geth' }} \ + -e NETWORK=hoodi \ + ghcr.io/dappnode/staker-test-util/snapshot-checker:latest + + # Option 2: Run as binary (alternative approach) + # Uncomment to use binary instead of Docker + # + # snapshot-check-binary: + # runs-on: self-hosted + # steps: + # - name: Download snapshot-checker binary + # run: | + # curl -L -o snapshot-checker \ + # https://github.com/dappnode/staker-test-util/releases/latest/download/snapshot-checker-linux-amd64 + # chmod +x snapshot-checker + # + # - name: Run Snapshot Checker + # run: | + # ./snapshot-checker \ + # --execution-client=${{ github.event.inputs.execution_client || 'geth' }} \ diff --git a/examples/workflows/staker-test.yml b/examples/workflows/staker-test.yml new file mode 100644 index 0000000..31f4040 --- /dev/null +++ b/examples/workflows/staker-test.yml @@ -0,0 +1,52 @@ +# Example: Using test-runner in your DAppNode package CI +# +# This workflow runs the staker test-runner against your package. +# Copy this to your package repo's .github/workflows/ directory. +# +# Prerequisites: +# - Self-hosted runner with Docker and access to dncore_network +# - IPFS hash of your package available + +name: Staker Test + +on: + pull_request: + branches: + - main + workflow_dispatch: + inputs: + ipfs_hash: + description: 'IPFS hash of the package to test' + required: true + +jobs: + staker-test: + runs-on: self-hosted + steps: + - name: Get IPFS hash + id: ipfs + run: | + # Use input if provided, otherwise you'll need to build/publish your package first + if [ -n "${{ github.event.inputs.ipfs_hash }}" ]; then + echo "hash=${{ github.event.inputs.ipfs_hash }}" >> $GITHUB_OUTPUT + else + # Add your logic to get IPFS hash from a previous build step + echo "hash=$IPFS_HASH" >> $GITHUB_OUTPUT + fi + + - name: Run Staker Test Runner + run: | + docker run --rm \ + --network dncore_network \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /var/lib/docker/volumes:/var/lib/docker/volumes \ + -e IPFS_GATEWAY_URL=http://ipfs.dappnode:8080 \ + -e IPFS_HASH=${{ steps.ipfs.outputs.hash }} \ + -e EXECUTION_CLIENT=geth \ + -e CONSENSUS_CLIENT=nimbus \ + -e GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} \ + -e GITHUB_REPOSITORY=${{ github.repository }} \ + -e GITHUB_PR_NUMBER=${{ github.event.pull_request.number }} \ + -e GITHUB_RUN_ID=${{ github.run_id }} \ + -e GITHUB_SERVER_URL=${{ github.server_url }} \ + ghcr.io/dappnode/staker-test-util/test-runner:latest diff --git a/internal/adapters/apis/beaconchain/beaconchain_adapter.go b/internal/adapters/apis/beaconchain/beaconchain_adapter.go index 7168d55..38d9276 100644 --- a/internal/adapters/apis/beaconchain/beaconchain_adapter.go +++ b/internal/adapters/apis/beaconchain/beaconchain_adapter.go @@ -2,7 +2,6 @@ package beaconchain import ( "bytes" - "clients-test/internal/logger" "context" "encoding/json" "fmt" @@ -15,7 +14,6 @@ import ( type BeaconchainAdapter struct { beaconChainUrl string client *http.Client - logPrefix string } // NewBeaconchainAdapter creates a new BeaconchainAdapter @@ -23,27 +21,22 @@ func NewBeaconchainAdapter(beaconChainUrl string) *BeaconchainAdapter { return &BeaconchainAdapter{ beaconChainUrl: beaconChainUrl, client: &http.Client{}, - logPrefix: "BeaconchainAdapter", } } // GetIsSyncing retrieves the syncing status from the beacon node with context func (b *BeaconchainAdapter) GetIsSyncing(ctx context.Context) (bool, error) { url := fmt.Sprintf("%s/eth/v1/node/syncing", b.beaconChainUrl) - logger.DebugWithPrefix(b.logPrefix, "GetIsSyncing: url=%s", url) req, err := http.NewRequestWithContext(ctx, "GET", url, nil) if err != nil { - logger.ErrorWithPrefix(b.logPrefix, "GetIsSyncing: failed to create request: %v", err) return false, err } resp, err := b.client.Do(req) if err != nil { - logger.ErrorWithPrefix(b.logPrefix, "GetIsSyncing: request failed: %v", err) return false, err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - logger.ErrorWithPrefix(b.logPrefix, "GetIsSyncing: non-200 status: %s", resp.Status) return false, fmt.Errorf("beacon node syncing failed: %s", resp.Status) } var result struct { @@ -52,10 +45,8 @@ func (b *BeaconchainAdapter) GetIsSyncing(ctx context.Context) (bool, error) { } `json:"data"` } if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { - logger.ErrorWithPrefix(b.logPrefix, "GetIsSyncing: failed to decode response: %v", err) return false, err } - logger.DebugWithPrefix(b.logPrefix, "GetIsSyncing: result=%+v", result.Data.IsSyncing) return result.Data.IsSyncing, nil } @@ -73,36 +64,29 @@ type blockHeaderResponse struct { // getBlockHeader retrieves the block header for a given block ID func (b *BeaconchainAdapter) getBlockHeader(ctx context.Context, blockID string) (*blockHeaderResponse, error) { url := fmt.Sprintf("%s/eth/v1/beacon/headers/%s", b.beaconChainUrl, blockID) - logger.DebugWithPrefix(b.logPrefix, "getBlockHeader: url=%s", url) req, err := http.NewRequestWithContext(ctx, "GET", url, nil) if err != nil { - logger.ErrorWithPrefix(b.logPrefix, "getBlockHeader: failed to create request: %v", err) return nil, fmt.Errorf("failed to send request to Beaconchain at %s: %w", url, err) } resp, err := b.client.Do(req) if err != nil { - logger.ErrorWithPrefix(b.logPrefix, "getBlockHeader: request failed: %v", err) return nil, err } defer resp.Body.Close() var result blockHeaderResponse if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { - logger.ErrorWithPrefix(b.logPrefix, "getBlockHeader: failed to decode response: %v", err) return nil, fmt.Errorf("failed to decode response for GetBlockHeader: %w", err) } - logger.DebugWithPrefix(b.logPrefix, "getBlockHeader: slot=%s", result.Data.Header.Message.Slot) return &result, nil } func (b *BeaconchainAdapter) getEpochHead(ctx context.Context, blockID string) (uint64, error) { header, err := b.getBlockHeader(ctx, blockID) if err != nil { - logger.ErrorWithPrefix(b.logPrefix, "getEpochHead: failed to get block header for blockID %s: %v", blockID, err) return 0, fmt.Errorf("failed to get block header for blockID %s: %w", blockID, err) } slot := header.Data.Header.Message.Slot epoch := getEpochFromSlot(slot) - logger.DebugWithPrefix(b.logPrefix, "getEpochHead: slot=%s epoch=%d", slot, epoch) return epoch, nil } @@ -122,32 +106,26 @@ func parseInt(slot string) uint64 { func (b *BeaconchainAdapter) GetValidatorLiveness(ctx context.Context, indexes []string) (map[string]bool, error) { epoch, err := b.getEpochHead(ctx, "head") if err != nil { - logger.ErrorWithPrefix(b.logPrefix, "GetValidatorLiveness: failed to get epoch head: %v", err) return nil, err } // Prepare POST request body as JSON array of indices jsonBytes, err := json.Marshal(indexes) if err != nil { - logger.ErrorWithPrefix(b.logPrefix, "GetValidatorLiveness: failed to marshal request body: %v", err) return nil, err } url := fmt.Sprintf("%s/eth/v1/validator/liveness/%d", b.beaconChainUrl, epoch) - logger.DebugWithPrefix(b.logPrefix, "GetValidatorLiveness: url=%s body=%v", url, indexes) req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewReader(jsonBytes)) if err != nil { - logger.ErrorWithPrefix(b.logPrefix, "GetValidatorLiveness: failed to create request: %v", err) return nil, err } req.Header.Set("Content-Type", "application/json") resp, err := b.client.Do(req) if err != nil { - logger.ErrorWithPrefix(b.logPrefix, "GetValidatorLiveness: request failed: %v", err) return nil, err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - logger.ErrorWithPrefix(b.logPrefix, "GetValidatorLiveness: non-200 status: %s", resp.Status) return nil, fmt.Errorf("validator liveness failed: %s", resp.Status) } var result struct { @@ -157,21 +135,50 @@ func (b *BeaconchainAdapter) GetValidatorLiveness(ctx context.Context, indexes [ } `json:"data"` } if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { - logger.ErrorWithPrefix(b.logPrefix, "GetValidatorLiveness: failed to decode response: %v", err) return nil, err } liveness := make(map[string]bool) for _, v := range result.Data { liveness[v.Index] = v.IsLive } - logger.DebugWithPrefix(b.logPrefix, "GetValidatorLiveness: liveness=%+v", liveness) return liveness, nil } +// GetClientVersion retrieves the version of the beacon node client +// See: https://ethereum.github.io/beacon-APIs/#/Node/getNodeVersion +func (b *BeaconchainAdapter) GetClientVersion(ctx context.Context) (string, error) { + url := fmt.Sprintf("%s/eth/v1/node/version", b.beaconChainUrl) + + req, err := http.NewRequestWithContext(ctx, "GET", url, nil) + if err != nil { + return "", err + } + + resp, err := b.client.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("beacon node version failed: %s", resp.Status) + } + + var result struct { + Data struct { + Version string `json:"version"` + } `json:"data"` + } + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return "", err + } + + return result.Data.Version, nil +} + // GetValidatorsIndexes retrieves the validator index for each given pubkey with status active_ongoing func (b *BeaconchainAdapter) GetValidatorsIndexes(ctx context.Context, pubkeys []string) ([]string, error) { url := fmt.Sprintf("%s/eth/v1/beacon/states/finalized/validators", b.beaconChainUrl) - logger.DebugWithPrefix(b.logPrefix, "GetValidatorsIndexes: url=%s pubkeys=%+v", url, pubkeys) requestBody := struct { IDs []string `json:"ids"` Statuses []string `json:"statuses"` @@ -181,26 +188,22 @@ func (b *BeaconchainAdapter) GetValidatorsIndexes(ctx context.Context, pubkeys [ } jsonBytes, err := json.Marshal(requestBody) if err != nil { - logger.ErrorWithPrefix(b.logPrefix, "GetValidatorsIndexes: failed to marshal request body: %v", err) return nil, err } req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewReader(jsonBytes)) if err != nil { - logger.ErrorWithPrefix(b.logPrefix, "GetValidatorsIndexes: failed to create request: %v", err) return nil, err } req.Header.Set("Content-Type", "application/json") resp, err := b.client.Do(req) if err != nil { - logger.ErrorWithPrefix(b.logPrefix, "GetValidatorsIndexes: request failed: %v", err) return nil, err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - logger.ErrorWithPrefix(b.logPrefix, "GetValidatorsIndexes: non-200 status: %s", resp.Status) return nil, fmt.Errorf("get validators indexes failed: %s", resp.Status) } @@ -213,13 +216,11 @@ func (b *BeaconchainAdapter) GetValidatorsIndexes(ctx context.Context, pubkeys [ } `json:"data"` } if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { - logger.ErrorWithPrefix(b.logPrefix, "GetValidatorsIndexes: failed to decode response: %v", err) return nil, err } indexes := make([]string, len(result.Data)) for i, v := range result.Data { indexes[i] = v.Index } - logger.DebugWithPrefix(b.logPrefix, "GetValidatorsIndexes: indexes=%+v", indexes) return indexes, nil } diff --git a/internal/adapters/apis/brain/brain_adapter.go b/internal/adapters/apis/brain/brain_adapter.go index 6f7021f..407f06e 100644 --- a/internal/adapters/apis/brain/brain_adapter.go +++ b/internal/adapters/apis/brain/brain_adapter.go @@ -1,7 +1,6 @@ package brain import ( - "clients-test/internal/logger" "context" "encoding/json" "fmt" @@ -12,51 +11,42 @@ import ( // It interacts with the brain service to fetch validator indexes // Example endpoint: /api/v0/brain/validators?tag=solo&format=index type BrainAdapter struct { - brainUrl string - client *http.Client - logPrefix string + brainUrl string + client *http.Client } // NewBrainAdapter creates a new BrainAdapter func NewBrainAdapter(brainUrl string) *BrainAdapter { return &BrainAdapter{ - brainUrl: brainUrl, - client: &http.Client{}, - logPrefix: "BrainAdapter", + brainUrl: brainUrl, + client: &http.Client{}, } } // GetValidatorsPubkeys fetches the validator public keys from the brain service with context func (b *BrainAdapter) GetValidatorsPubkeys(ctx context.Context) ([]string, error) { url := fmt.Sprintf("%s/api/v0/brain/validators?tag=solo&format=pubkey", b.brainUrl) - logger.DebugWithPrefix(b.logPrefix, "GetValidatorsPubkeys: url=%s", url) req, err := http.NewRequestWithContext(ctx, "GET", url, nil) if err != nil { - logger.ErrorWithPrefix(b.logPrefix, "GetValidatorsPubkeys: failed to create request: %v", err) return nil, err } resp, err := b.client.Do(req) if err != nil { - logger.ErrorWithPrefix(b.logPrefix, "GetValidatorsPubkeys: request failed: %v", err) return nil, err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - logger.ErrorWithPrefix(b.logPrefix, "GetValidatorsPubkeys: non-200 status: %s", resp.Status) return nil, fmt.Errorf("brain service error: %s", resp.Status) } var tagValidators map[string][]string if err := json.NewDecoder(resp.Body).Decode(&tagValidators); err != nil { - logger.ErrorWithPrefix(b.logPrefix, "GetValidatorsPubkeys: failed to decode response: %v", err) return nil, err } pubkeys, ok := tagValidators["solo"] if !ok { - logger.ErrorWithPrefix(b.logPrefix, "GetValidatorsPubkeys: no 'solo' tag found in response") return nil, fmt.Errorf("no 'solo' tag found in response") } - logger.DebugWithPrefix(b.logPrefix, "GetValidatorsPubkeys: pubkeys=%+v", pubkeys) return pubkeys, nil } diff --git a/internal/adapters/apis/dappmanager/dappmanager_adapter.go b/internal/adapters/apis/dappmanager/dappmanager_adapter.go index 21d9f89..20051b9 100644 --- a/internal/adapters/apis/dappmanager/dappmanager_adapter.go +++ b/internal/adapters/apis/dappmanager/dappmanager_adapter.go @@ -3,7 +3,6 @@ package dappmanager import ( "bytes" "clients-test/internal/application/domain" - "clients-test/internal/logger" "context" "encoding/json" "fmt" @@ -14,40 +13,33 @@ import ( // DappManagerAdapter is the adapter to interact with the DappManager API type DappManagerAdapter struct { - baseURL string - client *http.Client - logPrefix string + baseURL string + client *http.Client } // NewDappManagerAdapter creates a new DappManagerAdapter func NewDappManagerAdapter() *DappManagerAdapter { return &DappManagerAdapter{ - baseURL: "http://my.dappnode:7000", - client: &http.Client{}, - logPrefix: "DappManagerAdapter", + baseURL: "http://my.dappnode:7000", + client: &http.Client{}, } } // Ping sends a ping request to the DappManager API with context func (d *DappManagerAdapter) Ping(ctx context.Context) error { - logger.DebugWithPrefix(d.logPrefix, "Ping: url=%s", d.baseURL+"/ping") req, err := http.NewRequestWithContext(ctx, "GET", d.baseURL+"/ping", nil) if err != nil { - logger.ErrorWithPrefix(d.logPrefix, "Ping: failed to create request: %v", err) return err } resp, err := d.client.Do(req) if err != nil { - logger.ErrorWithPrefix(d.logPrefix, "Ping: request failed: %v", err) return err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - logger.ErrorWithPrefix(d.logPrefix, "Ping: non-200 status: %s", resp.Status) return fmt.Errorf("ping failed: %s", resp.Status) } - logger.DebugWithPrefix(d.logPrefix, "Ping: success") return nil } @@ -55,27 +47,22 @@ func (d *DappManagerAdapter) Ping(ctx context.Context) error { func (d *DappManagerAdapter) PackageInstall(ctx context.Context, pkg domain.Pkg) error { url := d.baseURL + "/packageInstall" payload := fmt.Sprintf(`{"name": "%s", "version": "%s", "userSettings": {}, "options": {"BYPASS_CORE_RESTRICTION": true, "BYPASS_SIGNED_RESTRICTION": true}}`, pkg.DnpName, pkg.Version) - logger.DebugWithPrefix(d.logPrefix, "PackageInstall: url=%s payload=%s", url, payload) req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewReader([]byte(payload))) if err != nil { - logger.ErrorWithPrefix(d.logPrefix, "PackageInstall: failed to create request: %v", err) return err } req.Header.Set("Content-Type", "application/json") resp, err := d.client.Do(req) if err != nil { - logger.ErrorWithPrefix(d.logPrefix, "PackageInstall: request failed: %v", err) return err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - logger.ErrorWithPrefix(d.logPrefix, "PackageInstall: non-200 status: %s", resp.Status) return fmt.Errorf("package install failed: %s", resp.Status) } - logger.DebugWithPrefix(d.logPrefix, "PackageInstall: success for pkg=%+v", pkg) return nil } @@ -97,40 +84,33 @@ type StakerConfigGetMinimal struct { func (d *DappManagerAdapter) GetStakerConfig(ctx context.Context, network string) (StakerConfigGetMinimal, error) { url := d.baseURL + "/stakerConfigGet" payload := fmt.Sprintf(`{"network": "%s"}`, network) - logger.DebugWithPrefix(d.logPrefix, "GetStakerConfig: url=%s payload=%s", url, payload) req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewReader([]byte(payload))) if err != nil { - logger.ErrorWithPrefix(d.logPrefix, "GetStakerConfig: failed to create request: %v", err) return StakerConfigGetMinimal{}, err } req.Header.Set("Content-Type", "application/json") resp, err := d.client.Do(req) if err != nil { - logger.ErrorWithPrefix(d.logPrefix, "GetStakerConfig: request failed: %v", err) return StakerConfigGetMinimal{}, err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - logger.ErrorWithPrefix(d.logPrefix, "GetStakerConfig: non-200 status: %s", resp.Status) return StakerConfigGetMinimal{}, fmt.Errorf("get staker config failed: %s", resp.Status) } bodyBytes, err := io.ReadAll(resp.Body) if err != nil { - logger.ErrorWithPrefix(d.logPrefix, "GetStakerConfig: failed to read response body: %v", err) return StakerConfigGetMinimal{}, err } var result StakerConfigGetMinimal err = json.Unmarshal(bodyBytes, &result) if err != nil { - logger.ErrorWithPrefix(d.logPrefix, "GetStakerConfig: failed to unmarshal response: %v", err) return StakerConfigGetMinimal{}, err } - logger.DebugWithPrefix(d.logPrefix, "GetStakerConfig: result=%+v", result) return result, nil } @@ -149,30 +129,24 @@ func (d *DappManagerAdapter) SetStakerConfig(ctx context.Context, stakerClients } jsonBytes, err := json.Marshal(payload) if err != nil { - logger.ErrorWithPrefix(d.logPrefix, "SetStakerConfig: failed to marshal payload: %v", err) return err } - logger.DebugWithPrefix(d.logPrefix, "SetStakerConfig: url=%s payload=%s", url, string(jsonBytes)) req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewReader(jsonBytes)) if err != nil { - logger.ErrorWithPrefix(d.logPrefix, "SetStakerConfig: failed to create request: %v", err) return err } req.Header.Set("Content-Type", "application/json") resp, err := d.client.Do(req) if err != nil { - logger.ErrorWithPrefix(d.logPrefix, "SetStakerConfig: request failed: %v", err) return err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - logger.ErrorWithPrefix(d.logPrefix, "SetStakerConfig: non-200 status: %s", resp.Status) return fmt.Errorf("set staker config failed: %s", resp.Status) } - logger.DebugWithPrefix(d.logPrefix, "SetStakerConfig: success for stakerClients=%+v", stakerClients) return nil } @@ -189,30 +163,24 @@ func (d *DappManagerAdapter) removePackage(ctx context.Context, dnpName string, } jsonBytes, err := json.Marshal(body) if err != nil { - logger.ErrorWithPrefix(d.logPrefix, "removePackage: failed to marshal body: %v", err) return err } - logger.DebugWithPrefix(d.logPrefix, "removePackage: url=%s body=%s", url, string(jsonBytes)) req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewReader(jsonBytes)) if err != nil { - logger.ErrorWithPrefix(d.logPrefix, "removePackage: failed to create request: %v", err) return err } req.Header.Set("Content-Type", "application/json") resp, err := d.client.Do(req) if err != nil { - logger.ErrorWithPrefix(d.logPrefix, "removePackage: request failed: %v", err) return err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - logger.ErrorWithPrefix(d.logPrefix, "removePackage: non-200 status: %s", resp.Status) return fmt.Errorf("remove package failed: %s", resp.Status) } - logger.DebugWithPrefix(d.logPrefix, "removePackage: success for dnpName=%s", dnpName) return nil } @@ -225,35 +193,29 @@ type installedPackageMinimal struct { // getPackages retrieves the list of installed packages from the DappManager API with context func (d *DappManagerAdapter) getPackages(ctx context.Context) ([]installedPackageMinimal, error) { url := d.baseURL + "/packagesGet" - logger.DebugWithPrefix(d.logPrefix, "getPackages: url=%s", url) req, err := http.NewRequestWithContext(ctx, "GET", url, nil) if err != nil { - logger.ErrorWithPrefix(d.logPrefix, "getPackages: failed to create request: %v", err) return nil, err } resp, err := d.client.Do(req) if err != nil { - logger.ErrorWithPrefix(d.logPrefix, "getPackages: request failed: %v", err) return nil, err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - logger.ErrorWithPrefix(d.logPrefix, "getPackages: non-200 status: %s", resp.Status) return nil, fmt.Errorf("get packages failed: %s", resp.Status) } bodyBytes, err := io.ReadAll(resp.Body) if err != nil { - logger.ErrorWithPrefix(d.logPrefix, "getPackages: failed to read response body: %v", err) return nil, err } var allPackages []map[string]interface{} err = json.Unmarshal(bodyBytes, &allPackages) if err != nil { - logger.ErrorWithPrefix(d.logPrefix, "getPackages: failed to unmarshal response: %v", err) return nil, err } @@ -266,41 +228,31 @@ func (d *DappManagerAdapter) getPackages(ctx context.Context) ([]installedPackag IsCore: isCore, }) } - logger.DebugWithPrefix(d.logPrefix, "getPackages: result=%+v", result) return result, nil } // RemoveNonCorePackages removes all non-core packages from the Dappnode to clean up the system with context -func (d *DappManagerAdapter) RemoveNonCorePackages(ctx context.Context) []error { - logger.DebugWithPrefix(d.logPrefix, "RemoveNonCorePackages: called") +// Returns a list of packages that were skipped and any errors encountered +func (d *DappManagerAdapter) RemoveNonCorePackages(ctx context.Context) (skipped []string, errors []error) { packages, err := d.getPackages(ctx) if err != nil { - logger.ErrorWithPrefix(d.logPrefix, "RemoveNonCorePackages: failed to get packages: %v", err) - return []error{err} + return nil, []error{err} } - var errors []error for _, pkg := range packages { if !pkg.IsCore { // skip if pkg.DnpName includes web3signer or mev-boost if strings.Contains(pkg.DnpName, "web3signer") || strings.Contains(pkg.DnpName, "mev-boost") { - logger.DebugWithPrefix(d.logPrefix, "RemoveNonCorePackages: skipping package %s as it is web3signer or mev-boost", pkg.DnpName) + skipped = append(skipped, pkg.DnpName) continue } deleteVolumes := true - logger.DebugWithPrefix(d.logPrefix, "RemoveNonCorePackages: removing dnpName=%s deleteVolumes=%v", pkg.DnpName, deleteVolumes) err := d.removePackage(ctx, pkg.DnpName, &deleteVolumes) if err != nil { - logger.ErrorWithPrefix(d.logPrefix, "RemoveNonCorePackages: failed to remove package %s: %v", pkg.DnpName, err) errors = append(errors, fmt.Errorf("failed to remove package %s: %w", pkg.DnpName, err)) continue } } } - if len(errors) > 0 { - return errors - } - - logger.DebugWithPrefix(d.logPrefix, "RemoveNonCorePackages: success") - return nil + return skipped, errors } diff --git a/internal/adapters/apis/docker/docker_adapter.go b/internal/adapters/apis/docker/docker_adapter.go index b437e44..75f21f4 100644 --- a/internal/adapters/apis/docker/docker_adapter.go +++ b/internal/adapters/apis/docker/docker_adapter.go @@ -1,18 +1,24 @@ package docker import ( + "bufio" "context" "fmt" - - "clients-test/internal/logger" + "io" + "os" + "strings" + "time" "github.com/docker/docker/api/types/container" + "github.com/docker/docker/api/types/filters" + "github.com/docker/docker/api/types/image" + "github.com/docker/docker/api/types/mount" "github.com/docker/docker/client" + "github.com/docker/docker/pkg/stdcopy" ) type DockerAdapter struct { - cli *client.Client - logPrefix string + cli *client.Client } func NewDockerAdapter() (*DockerAdapter, error) { @@ -20,15 +26,13 @@ func NewDockerAdapter() (*DockerAdapter, error) { if err != nil { return nil, err } - return &DockerAdapter{cli: cli, logPrefix: "DockerAdapter"}, nil + return &DockerAdapter{cli: cli}, nil } // StopAndGetVolumeTarget stops the container, checks for a single volume, and returns the volume target path func (d *DockerAdapter) StopAndGetVolumeTarget(ctx context.Context, containerName string, containerVolumeName string) (string, error) { - logger.DebugWithPrefix(d.logPrefix, "StopAndGetVolumeTarget: inspecting container %s", containerName) containerJSON, err := d.cli.ContainerInspect(ctx, containerName) if err != nil { - logger.ErrorWithPrefix(d.logPrefix, "StopAndGetVolumeTarget: failed to inspect container: %v", err) return "", fmt.Errorf("failed to inspect container: %w", err) } @@ -40,15 +44,11 @@ func (d *DockerAdapter) StopAndGetVolumeTarget(ctx context.Context, containerNam volumeName := mount.Name volumeTarget := fmt.Sprintf("/var/lib/docker/volumes/%s/_data", volumeName) - logger.DebugWithPrefix(d.logPrefix, "StopAndGetVolumeTarget: volumeName=%s volumeTarget=%s", volumeName, volumeTarget) // Stop the container - logger.DebugWithPrefix(d.logPrefix, "StopAndGetVolumeTarget: stopping container %s", containerName) if err := d.cli.ContainerStop(ctx, containerName, container.StopOptions{}); err != nil { - logger.ErrorWithPrefix(d.logPrefix, "StopAndGetVolumeTarget: failed to stop container: %v", err) return "", fmt.Errorf("failed to stop container: %w", err) } - logger.DebugWithPrefix(d.logPrefix, "StopAndGetVolumeTarget: stopped container %s", containerName) return volumeTarget, nil } @@ -59,11 +59,293 @@ func (d *DockerAdapter) StopAndGetVolumeTarget(ctx context.Context, containerNam // StartContainer starts the given container func (d *DockerAdapter) StartContainer(ctx context.Context, containerName string) error { - logger.DebugWithPrefix(d.logPrefix, "StartContainer: starting container %s", containerName) if err := d.cli.ContainerStart(ctx, containerName, container.StartOptions{}); err != nil { - logger.ErrorWithPrefix(d.logPrefix, "StartContainer: failed to start container: %v", err) return fmt.Errorf("failed to start container: %w", err) } - logger.DebugWithPrefix(d.logPrefix, "StartContainer: started container %s", containerName) + return nil +} + +// GetContainerErrorLogs retrieves error logs from a container within a time range +// Returns up to maxLines error lines (lines containing "error", "err", "fatal", "panic", etc.) +func (d *DockerAdapter) GetContainerErrorLogs(ctx context.Context, containerName string, since, until time.Time, maxLines int) ([]string, error) { + // Check if container exists first + _, err := d.cli.ContainerInspect(ctx, containerName) + if err != nil { + return nil, nil // Return nil instead of error - container might not exist yet + } + + options := container.LogsOptions{ + ShowStdout: true, + ShowStderr: true, + Since: since.Format(time.RFC3339), + Until: until.Format(time.RFC3339), + Timestamps: true, + } + + reader, err := d.cli.ContainerLogs(ctx, containerName, options) + if err != nil { + return nil, nil // Return nil instead of error - might be transient + } + defer reader.Close() + + errorLines := make([]string, 0) + errorKeywords := []string{"error", "err:", "fatal", "panic", "exception", "failed", "failure"} + + // Docker logs have an 8-byte header for multiplexed streams + // We need to handle this properly + buf := make([]byte, 8) + for { + // Read the header + _, err := io.ReadFull(reader, buf) + if err != nil { + if err == io.EOF { + break + } + // Try reading as plain text if header read fails + break + } + + // Get the size of the log line from header bytes 4-7 + size := int(buf[4])<<24 | int(buf[5])<<16 | int(buf[6])<<8 | int(buf[7]) + + // Read the actual log line + line := make([]byte, size) + _, err = io.ReadFull(reader, line) + if err != nil { + break + } + + lineStr := strings.TrimSpace(string(line)) + if lineStr == "" { + continue + } + + // Check if line contains error keywords (case-insensitive) + lineLower := strings.ToLower(lineStr) + for _, keyword := range errorKeywords { + if strings.Contains(lineLower, keyword) { + // Truncate very long lines + if len(lineStr) > 500 { + lineStr = lineStr[:500] + "..." + } + errorLines = append(errorLines, lineStr) + if len(errorLines) >= maxLines { + return errorLines, nil + } + break + } + } + } + + // If the multiplexed read failed, try reading as plain text + if len(errorLines) == 0 { + reader2, err := d.cli.ContainerLogs(ctx, containerName, options) + if err != nil { + return nil, nil + } + defer reader2.Close() + + scanner := bufio.NewScanner(reader2) + for scanner.Scan() { + lineStr := strings.TrimSpace(scanner.Text()) + if lineStr == "" { + continue + } + + lineLower := strings.ToLower(lineStr) + for _, keyword := range errorKeywords { + if strings.Contains(lineLower, keyword) { + if len(lineStr) > 500 { + lineStr = lineStr[:500] + "..." + } + errorLines = append(errorLines, lineStr) + if len(errorLines) >= maxLines { + return errorLines, nil + } + break + } + } + } + } + + return errorLines, nil +} + +// CollectAllContainerErrorLogs collects error logs from multiple containers +func (d *DockerAdapter) CollectAllContainerErrorLogs(ctx context.Context, containerNames []string, since, until time.Time, maxLinesPerContainer int) map[string][]string { + result := make(map[string][]string) + + for _, name := range containerNames { + if name == "" { + continue + } + lines, err := d.GetContainerErrorLogs(ctx, name, since, until, maxLinesPerContainer) + if err != nil { + continue + } + if len(lines) > 0 { + result[name] = lines + } + } + + return result +} + +// StopContainer stops a container by name +func (d *DockerAdapter) StopContainer(ctx context.Context, containerName string) error { + // Check if container exists first + _, err := d.cli.ContainerInspect(ctx, containerName) + if err != nil { + return nil // Return nil - container might not exist + } + + if err := d.cli.ContainerStop(ctx, containerName, container.StopOptions{}); err != nil { + return fmt.Errorf("failed to stop container: %w", err) + } + + return nil +} + +// ListContainersByPrefix returns a list of container IDs that match the given name prefix +func (d *DockerAdapter) ListContainersByPrefix(ctx context.Context, prefix string) ([]string, error) { + filterArgs := filters.NewArgs() + filterArgs.Add("name", prefix) + + containers, err := d.cli.ContainerList(ctx, container.ListOptions{ + Filters: filterArgs, + }) + if err != nil { + return nil, fmt.Errorf("failed to list containers: %w", err) + } + + var ids []string + for _, c := range containers { + ids = append(ids, c.ID) + } + + return ids, nil +} + +// StopContainerWithTimeout stops a container by ID with a specific timeout in seconds +func (d *DockerAdapter) StopContainerWithTimeout(ctx context.Context, containerID string, timeoutSeconds int) error { + timeout := timeoutSeconds + if err := d.cli.ContainerStop(ctx, containerID, container.StopOptions{Timeout: &timeout}); err != nil { + return fmt.Errorf("failed to stop container: %w", err) + } + + return nil +} + +const ( + alpineImage = "alpine:latest" +) + +// RunSnapshotDownload runs an Alpine container to download and extract a snapshot +// Uses aria2c for parallel HTTP range request downloads and zstd for decompression +// Returns the duration of the download operation +func (d *DockerAdapter) RunSnapshotDownload(ctx context.Context, containerName, clientName, network, targetPath, baseURL string) error { + // Pull alpine image if not present + if err := d.ensureImage(ctx, alpineImage); err != nil { + return fmt.Errorf("failed to ensure alpine image: %w", err) + } + + // Build the shell script for download and extraction + // Optimizations: + // - aria2c -x16 -s16: 16 parallel connections using HTTP range requests + // - zstd -T0: multi-threaded decompression using all CPU cores + // - No -v flag on tar: avoid printing every filename (major speedup) + shellScript := fmt.Sprintf(` +set -e +apk add --no-cache aria2 tar zstd pv bash curl > /dev/null 2>&1 +BLOCK_NUMBER=$(curl -sf %s/%s/%s/latest) +SNAPSHOT_URL="%s/%s/%s/${BLOCK_NUMBER}/snapshot.tar.zst" +echo "[%s] Downloading snapshot for block number: ${BLOCK_NUMBER}" +echo "[%s] Using 16 parallel connections with HTTP range requests" +aria2c -x16 -s16 --file-allocation=none --console-log-level=warn --summary-interval=30 --show-console-readout=false -d /data -o snapshot.tar.zst "${SNAPSHOT_URL}" 2>&1 | awk '/^\[#/{print "[%s] " $0; fflush()}' +echo "[%s] Download complete. Extracting with $(nproc) CPU cores..." +bash -c 'pv -f -i 30 -N "%s" -ptebar /data/snapshot.tar.zst 2> >(tr "\r" "\n" >&2) | zstd -d -T0 | tar -xf - -C /data' +rm -f /data/snapshot.tar.zst +echo "[%s] Snapshot extraction complete" +`, baseURL, network, clientName, baseURL, network, clientName, clientName, clientName, clientName, clientName, clientName, clientName) + + // Create container config + config := &container.Config{ + Image: alpineImage, + Entrypoint: []string{"/bin/sh"}, + Cmd: []string{"-c", shellScript}, + Tty: false, + } + + // Create host config with volume mount + hostConfig := &container.HostConfig{ + Mounts: []mount.Mount{ + { + Type: mount.TypeBind, + Source: targetPath, + Target: "/data", + }, + }, + AutoRemove: true, + } + + // Create the container + resp, err := d.cli.ContainerCreate(ctx, config, hostConfig, nil, nil, containerName) + if err != nil { + return fmt.Errorf("failed to create container: %w", err) + } + + // Start the container + if err := d.cli.ContainerStart(ctx, resp.ID, container.StartOptions{}); err != nil { + return fmt.Errorf("failed to start container: %w", err) + } + + // Attach to container output for real-time logging + attachResp, err := d.cli.ContainerAttach(ctx, resp.ID, container.AttachOptions{ + Stream: true, + Stdout: true, + Stderr: true, + }) + if err == nil { + defer attachResp.Close() + // Copy output to stdout/stderr in a goroutine + go func() { + _, _ = stdcopy.StdCopy(os.Stdout, os.Stderr, attachResp.Reader) + }() + } + + // Wait for container to finish + statusCh, errCh := d.cli.ContainerWait(ctx, resp.ID, container.WaitConditionNotRunning) + + select { + case err := <-errCh: + if err != nil { + return fmt.Errorf("error waiting for container: %w", err) + } + case status := <-statusCh: + if status.StatusCode != 0 { + return fmt.Errorf("container exited with status %d", status.StatusCode) + } + } + + return nil +} + +// ensureImage pulls an image if it's not already present locally +func (d *DockerAdapter) ensureImage(ctx context.Context, imageName string) error { + // Check if image exists + _, err := d.cli.ImageInspect(ctx, imageName) + if err == nil { + return nil // Image already exists + } + + reader, err := d.cli.ImagePull(ctx, imageName, image.PullOptions{}) + if err != nil { + return fmt.Errorf("failed to pull image: %w", err) + } + defer reader.Close() + + // Discard the output but wait for pull to complete + _, _ = io.Copy(io.Discard, reader) + return nil } diff --git a/internal/adapters/apis/execution/execution_adapter.go b/internal/adapters/apis/execution/execution_adapter.go index e12e673..e6f7251 100644 --- a/internal/adapters/apis/execution/execution_adapter.go +++ b/internal/adapters/apis/execution/execution_adapter.go @@ -2,7 +2,6 @@ package execution import ( "bytes" - "clients-test/internal/logger" "context" "encoding/json" "fmt" @@ -13,24 +12,64 @@ import ( // It interacts with an Ethereum execution client via JSON-RPC // See: https://ethereum.github.io/execution-apis/api-documentation/ type ExecutionAdapter struct { - baseURL string - client *http.Client - logPrefix string + baseURL string + client *http.Client } // NewExecutionAdapter creates a new ExecutionAdapter func NewExecutionAdapter(baseURL string) *ExecutionAdapter { return &ExecutionAdapter{ - baseURL: baseURL, - client: &http.Client{}, - logPrefix: "ExecutionAdapter", + baseURL: baseURL, + client: &http.Client{}, } } +// GetClientVersion retrieves the version of the execution client +// See: https://ethereum.github.io/execution-apis/api-documentation/ (web3_clientVersion) +func (e *ExecutionAdapter) GetClientVersion(ctx context.Context) (string, error) { + url := e.baseURL + + // JSON-RPC request body for web3_clientVersion + body := map[string]interface{}{ + "jsonrpc": "2.0", + "method": "web3_clientVersion", + "params": []interface{}{}, + "id": 1, + } + jsonBytes, err := json.Marshal(body) + if err != nil { + return "", err + } + + req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewReader(jsonBytes)) + if err != nil { + return "", err + } + req.Header.Set("Content-Type", "application/json") + + resp, err := e.client.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("web3_clientVersion failed: %s", resp.Status) + } + + var rpcResp struct { + Result string `json:"result"` + } + if err := json.NewDecoder(resp.Body).Decode(&rpcResp); err != nil { + return "", err + } + + return rpcResp.Result, nil +} + // GetIsSyncing retrieves the syncing status from the execution client with context func (e *ExecutionAdapter) GetIsSyncing(ctx context.Context) (bool, error) { url := e.baseURL - logger.DebugWithPrefix(e.logPrefix, "GetIsSyncing: url=%s", url) // JSON-RPC request body for eth_syncing body := map[string]interface{}{ "jsonrpc": "2.0", @@ -40,26 +79,22 @@ func (e *ExecutionAdapter) GetIsSyncing(ctx context.Context) (bool, error) { } jsonBytes, err := json.Marshal(body) if err != nil { - logger.ErrorWithPrefix(e.logPrefix, "GetIsSyncing: failed to marshal body: %v", err) return false, err } req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewReader(jsonBytes)) if err != nil { - logger.ErrorWithPrefix(e.logPrefix, "GetIsSyncing: failed to create request: %v", err) return false, err } req.Header.Set("Content-Type", "application/json") resp, err := e.client.Do(req) if err != nil { - logger.ErrorWithPrefix(e.logPrefix, "GetIsSyncing: request failed: %v", err) return false, err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - logger.ErrorWithPrefix(e.logPrefix, "GetIsSyncing: non-200 status: %s", resp.Status) return false, fmt.Errorf("eth_syncing failed: %s", resp.Status) } @@ -67,11 +102,9 @@ func (e *ExecutionAdapter) GetIsSyncing(ctx context.Context) (bool, error) { Result interface{} `json:"result"` } if err := json.NewDecoder(resp.Body).Decode(&rpcResp); err != nil { - logger.ErrorWithPrefix(e.logPrefix, "GetIsSyncing: failed to decode response: %v", err) return false, err } - logger.DebugWithPrefix(e.logPrefix, "GetIsSyncing: result=%+v", rpcResp.Result) // Per spec, result is either false (not syncing) or an object (syncing) if b, ok := rpcResp.Result.(bool); ok { return b, nil diff --git a/internal/adapters/apis/github/github_adapter.go b/internal/adapters/apis/github/github_adapter.go new file mode 100644 index 0000000..7497fdc --- /dev/null +++ b/internal/adapters/apis/github/github_adapter.go @@ -0,0 +1,236 @@ +package github + +import ( + "bytes" + "clients-test/internal/application/domain" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strconv" + "strings" +) + +// GitHubConfig holds the configuration for GitHub API interactions +type GitHubConfig struct { + Token string // GitHub token (PAT or GITHUB_TOKEN from Actions) + Owner string // Repository owner (e.g., "dappnode") + Repo string // Repository name (e.g., "staker-test-util") + PRNumber int // Pull request number (0 if not in PR context) + RunID string // GitHub Actions run ID for linking to CI logs + RunURL string // Full URL to the GitHub Actions run + ServerURL string // GitHub server URL (for GitHub Enterprise) + Repository string // Full repository name (owner/repo) +} + +// GitHubAdapter handles interactions with the GitHub API +type GitHubAdapter struct { + config GitHubConfig + client *http.Client + baseURL string +} + +// NewGitHubAdapter creates a new GitHub adapter +func NewGitHubAdapter(config GitHubConfig) *GitHubAdapter { + baseURL := "https://api.github.com" + if config.ServerURL != "" && config.ServerURL != "https://github.com" { + // For GitHub Enterprise + baseURL = strings.TrimSuffix(config.ServerURL, "/") + "/api/v3" + } + + return &GitHubAdapter{ + config: config, + client: &http.Client{}, + baseURL: baseURL, + } +} + +// IsEnabled returns true if the GitHub adapter is properly configured +func (g *GitHubAdapter) IsEnabled() bool { + return g.config.Token != "" && g.config.Owner != "" && g.config.Repo != "" && g.config.PRNumber > 0 +} + +// IssueComment represents a GitHub issue/PR comment +type IssueComment struct { + ID int64 `json:"id"` + Body string `json:"body"` +} + +// CommentOnPR creates or updates a comment on a pull request with the test report +func (g *GitHubAdapter) CommentOnPR(ctx context.Context, report *domain.TestReport) error { + if !g.IsEnabled() { + return nil + } + + // Generate markdown report + markdown := report.ToMarkdown() + + // Add CI logs link if available + if g.config.RunURL != "" { + markdown += fmt.Sprintf("\n---\n📋 [View full CI logs](%s)\n", g.config.RunURL) + } + + // Add a signature to identify our comments for updates + signature := "\n\n" + markdown += signature + + // Check for existing comment to update + existingCommentID, err := g.findExistingComment(ctx, signature) + if err != nil { + // Continue with creating new comment if we can't find existing + existingCommentID = 0 + } + + if existingCommentID > 0 { + // Update existing comment + return g.updateComment(ctx, existingCommentID, markdown) + } + + // Create new comment + return g.createComment(ctx, markdown) +} + +// findExistingComment looks for an existing comment with our signature +func (g *GitHubAdapter) findExistingComment(ctx context.Context, signature string) (int64, error) { + url := fmt.Sprintf("%s/repos/%s/%s/issues/%d/comments", + g.baseURL, g.config.Owner, g.config.Repo, g.config.PRNumber) + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return 0, err + } + + g.setHeaders(req) + + resp, err := g.client.Do(req) + if err != nil { + return 0, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return 0, fmt.Errorf("failed to list comments: %s - %s", resp.Status, string(body)) + } + + var comments []IssueComment + if err := json.NewDecoder(resp.Body).Decode(&comments); err != nil { + return 0, err + } + + for _, comment := range comments { + if strings.Contains(comment.Body, signature) { + return comment.ID, nil + } + } + + return 0, nil +} + +// createComment creates a new comment on the PR +func (g *GitHubAdapter) createComment(ctx context.Context, body string) error { + url := fmt.Sprintf("%s/repos/%s/%s/issues/%d/comments", + g.baseURL, g.config.Owner, g.config.Repo, g.config.PRNumber) + + payload := map[string]string{"body": body} + jsonPayload, err := json.Marshal(payload) + if err != nil { + return err + } + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewBuffer(jsonPayload)) + if err != nil { + return err + } + + g.setHeaders(req) + req.Header.Set("Content-Type", "application/json") + + resp, err := g.client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusCreated { + body, _ := io.ReadAll(resp.Body) + return fmt.Errorf("failed to create comment: %s - %s", resp.Status, string(body)) + } + + return nil +} + +// updateComment updates an existing comment +func (g *GitHubAdapter) updateComment(ctx context.Context, commentID int64, body string) error { + url := fmt.Sprintf("%s/repos/%s/%s/issues/comments/%d", + g.baseURL, g.config.Owner, g.config.Repo, commentID) + + payload := map[string]string{"body": body} + jsonPayload, err := json.Marshal(payload) + if err != nil { + return err + } + + req, err := http.NewRequestWithContext(ctx, http.MethodPatch, url, bytes.NewBuffer(jsonPayload)) + if err != nil { + return err + } + + g.setHeaders(req) + req.Header.Set("Content-Type", "application/json") + + resp, err := g.client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return fmt.Errorf("failed to update comment: %s - %s", resp.Status, string(body)) + } + + return nil +} + +// setHeaders sets common headers for GitHub API requests +func (g *GitHubAdapter) setHeaders(req *http.Request) { + req.Header.Set("Authorization", "Bearer "+g.config.Token) + req.Header.Set("Accept", "application/vnd.github+json") + req.Header.Set("X-GitHub-Api-Version", "2022-11-28") +} + +// ParseGitHubConfigFromEnv creates a GitHubConfig from environment variables +// This is typically used in GitHub Actions context +func ParseGitHubConfigFromEnv(token, repository, prNumber, runID, serverURL string) GitHubConfig { + config := GitHubConfig{ + Token: token, + Repository: repository, + RunID: runID, + ServerURL: serverURL, + } + + // Parse repository into owner/repo + if repository != "" { + parts := strings.SplitN(repository, "/", 2) + if len(parts) == 2 { + config.Owner = parts[0] + config.Repo = parts[1] + } + } + + // Parse PR number + if prNumber != "" { + if num, err := strconv.Atoi(prNumber); err == nil { + config.PRNumber = num + } + } + + // Build run URL + if serverURL != "" && repository != "" && runID != "" { + config.RunURL = fmt.Sprintf("%s/%s/actions/runs/%s", serverURL, repository, runID) + } + + return config +} diff --git a/internal/adapters/apis/ipfs/ipfs_adapter.go b/internal/adapters/apis/ipfs/ipfs_adapter.go index 84dffdb..269e6f9 100644 --- a/internal/adapters/apis/ipfs/ipfs_adapter.go +++ b/internal/adapters/apis/ipfs/ipfs_adapter.go @@ -2,7 +2,6 @@ package ipfs import ( "clients-test/internal/application/domain" - "clients-test/internal/logger" "context" "encoding/json" "fmt" @@ -13,16 +12,14 @@ import ( ) type IPFSAdapter struct { - gateway *string - client *http.Client - logPrefix string + gateway *string + client *http.Client } func NewIPFSAdapter(gateway *string) *IPFSAdapter { return &IPFSAdapter{ - gateway: gateway, - client: &http.Client{}, - logPrefix: "IPFSAdapter", + gateway: gateway, + client: &http.Client{}, } } @@ -42,24 +39,19 @@ type composeFile struct { // GetDnpNameAndServiceName fetches dappnode_package.json and docker-compose.yml from the given IPFS directory hash // and returns the dnpName and service name. Use the internal functions func (a *IPFSAdapter) GetDnpNameAndServiceName(ctx context.Context, ipfsHash string) (domain.Pkg, error) { - logger.DebugWithPrefix(a.logPrefix, "GetDnpNameAndServiceName: ipfsHash=%s", ipfsHash) dnpName, err := a.getDnpNameFromHash(ctx, ipfsHash) if err != nil { - logger.ErrorWithPrefix(a.logPrefix, "GetDnpNameAndServiceName: failed to get dnpName: %v", err) return domain.Pkg{}, fmt.Errorf("failed to get dnpName from IPFS hash: %w", err) } serviceName, err := a.getComposeServiceName(ctx, ipfsHash) if err != nil { - logger.ErrorWithPrefix(a.logPrefix, "GetDnpNameAndServiceName: failed to get compose service name: %v", err) return domain.Pkg{}, fmt.Errorf("failed to get compose service name: %w", err) } // Fetch the root volume name from compose volumeName, err := a.getComposeVolumeName(ctx, ipfsHash) if err != nil { - logger.ErrorWithPrefix(a.logPrefix, "GetDnpNameAndServiceName: failed to get compose volume name: %v", err) return domain.Pkg{}, fmt.Errorf("failed to get compose volume name: %w", err) } - logger.DebugWithPrefix(a.logPrefix, "GetDnpNameAndServiceName: dnpName=%s serviceName=%s", dnpName, serviceName) return domain.Pkg{ DnpName: dnpName, ServiceName: serviceName, @@ -71,64 +63,51 @@ func (a *IPFSAdapter) GetDnpNameAndServiceName(ctx context.Context, ipfsHash str // getDnpNameFromHash fetches dappnode_package.json from the given IPFS directory hash and returns the dnpName value func (a *IPFSAdapter) getDnpNameFromHash(ctx context.Context, ipfsHash string) (string, error) { url := fmt.Sprintf("%s/ipfs/%s/dappnode_package.json", *a.gateway, ipfsHash) - logger.DebugWithPrefix(a.logPrefix, "getDnpNameFromHash: url=%s", url) req, err := http.NewRequestWithContext(ctx, "GET", url, nil) if err != nil { - logger.ErrorWithPrefix(a.logPrefix, "getDnpNameFromHash: failed to create request: %v", err) return "", err } resp, err := a.client.Do(req) if err != nil { - logger.ErrorWithPrefix(a.logPrefix, "getDnpNameFromHash: request failed: %v", err) return "", err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - logger.ErrorWithPrefix(a.logPrefix, "getDnpNameFromHash: non-200 status: %s", resp.Status) return "", fmt.Errorf("failed to fetch dappnode_package.json: %s", resp.Status) } body, err := io.ReadAll(resp.Body) if err != nil { - logger.ErrorWithPrefix(a.logPrefix, "getDnpNameFromHash: failed to read body: %v", err) return "", err } var pkg struct { DnpName string `json:"name"` } if err := json.Unmarshal(body, &pkg); err != nil { - logger.ErrorWithPrefix(a.logPrefix, "getDnpNameFromHash: failed to unmarshal: %v", err) return "", err } if pkg.DnpName == "" { - logger.ErrorWithPrefix(a.logPrefix, "getDnpNameFromHash: dnpName not found in dappnode_package.json") return "", fmt.Errorf("dnpName not found in dappnode_package.json") } - logger.DebugWithPrefix(a.logPrefix, "getDnpNameFromHash: dnpName=%s", pkg.DnpName) return pkg.DnpName, nil } // fetchComposeFile retrieves and decodes the docker-compose.yml into a composeFile func (a *IPFSAdapter) fetchComposeFile(ctx context.Context, ipfsHash string) (*composeFile, error) { url := fmt.Sprintf("%s/ipfs/%s/docker-compose.yml", *a.gateway, ipfsHash) - logger.DebugWithPrefix(a.logPrefix, "fetchComposeFile: url=%s", url) req, err := http.NewRequestWithContext(ctx, "GET", url, nil) if err != nil { - logger.ErrorWithPrefix(a.logPrefix, "fetchComposeFile: failed to create request: %v", err) return nil, err } resp, err := a.client.Do(req) if err != nil { - logger.ErrorWithPrefix(a.logPrefix, "fetchComposeFile: request failed: %v", err) return nil, err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - logger.ErrorWithPrefix(a.logPrefix, "fetchComposeFile: non-200 status: %s", resp.Status) return nil, fmt.Errorf("failed to fetch docker-compose.yml: %s", resp.Status) } var compose composeFile if err := yaml.NewDecoder(resp.Body).Decode(&compose); err != nil { - logger.ErrorWithPrefix(a.logPrefix, "fetchComposeFile: failed to decode yaml: %v", err) return nil, err } return &compose, nil @@ -141,14 +120,11 @@ func (a *IPFSAdapter) getComposeServiceName(ctx context.Context, ipfsHash string return "", err } if len(compose.Services) != 1 { - logger.ErrorWithPrefix(a.logPrefix, "getComposeServiceName: expected 1 service, got %d", len(compose.Services)) return "", fmt.Errorf("expected exactly one service in docker-compose.yml, got %d", len(compose.Services)) } for name := range compose.Services { - logger.DebugWithPrefix(a.logPrefix, "getComposeServiceName: serviceName=%s", name) return name, nil } - logger.ErrorWithPrefix(a.logPrefix, "getComposeServiceName: no service found in docker-compose.yml") return "", fmt.Errorf("no service found in docker-compose.yml") } @@ -159,13 +135,10 @@ func (a *IPFSAdapter) getComposeVolumeName(ctx context.Context, ipfsHash string) return "", err } if len(compose.Volumes) != 1 { - logger.ErrorWithPrefix(a.logPrefix, "getComposeVolumeName: expected 1 volume, got %d", len(compose.Volumes)) return "", fmt.Errorf("expected exactly one volume in docker-compose.yml, got %d", len(compose.Volumes)) } for name := range compose.Volumes { - logger.DebugWithPrefix(a.logPrefix, "getComposeVolumeName: volumeName=%s", name) return name, nil } - logger.ErrorWithPrefix(a.logPrefix, "getComposeVolumeName: no volume found in docker-compose.yml") return "", fmt.Errorf("no volume found in docker-compose.yml") } diff --git a/internal/adapters/apis/snapshots/snapshots_adapter.go b/internal/adapters/apis/snapshots/snapshots_adapter.go new file mode 100644 index 0000000..876f17d --- /dev/null +++ b/internal/adapters/apis/snapshots/snapshots_adapter.go @@ -0,0 +1,107 @@ +package snapshots + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" +) + +const ( + defaultBaseURL = "https://snapshots.ethpandaops.io" +) + +type SnapshotsAdapter struct { + baseURL string +} + +func NewSnapshotsAdapter() *SnapshotsAdapter { + return &SnapshotsAdapter{ + baseURL: defaultBaseURL, + } +} + +func NewSnapshotsAdapterWithURL(baseURL string) *SnapshotsAdapter { + return &SnapshotsAdapter{ + baseURL: baseURL, + } +} + +// GetBaseURL returns the base URL of the snapshots API +func (s *SnapshotsAdapter) GetBaseURL() string { + return s.baseURL +} + +// GetLatestBlockNumber fetches the latest available block number for a given network and client +func (s *SnapshotsAdapter) GetLatestBlockNumber(ctx context.Context, network, client string) (string, error) { + url := fmt.Sprintf("%s/%s/%s/latest", s.baseURL, network, client) + + req, err := http.NewRequestWithContext(ctx, "GET", url, nil) + if err != nil { + return "", fmt.Errorf("failed to create request: %w", err) + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return "", fmt.Errorf("failed to fetch latest block number: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("failed to fetch latest block number: status %s", resp.Status) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return "", fmt.Errorf("failed to read response body: %w", err) + } + + blockNumber := strings.TrimSpace(string(body)) + return blockNumber, nil +} + +// GetSnapshotURL returns the full URL for a snapshot file +func (s *SnapshotsAdapter) GetSnapshotURL(network, client, blockNumber string) string { + return fmt.Sprintf("%s/%s/%s/%s/snapshot.tar.zst", s.baseURL, network, client, blockNumber) +} + +// GetClientVersion fetches the client version used to generate the snapshot +// The version is retrieved from the _snapshot_web3_clientVersion.json file +func (s *SnapshotsAdapter) GetClientVersion(ctx context.Context, network, client, blockNumber string) (string, error) { + url := fmt.Sprintf("%s/%s/%s/%s/_snapshot_web3_clientVersion.json", s.baseURL, network, client, blockNumber) + + req, err := http.NewRequestWithContext(ctx, "GET", url, nil) + if err != nil { + return "", fmt.Errorf("failed to create request: %w", err) + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return "", fmt.Errorf("failed to fetch client version: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("failed to fetch client version: status %s", resp.Status) + } + + var result struct { + Result string `json:"result"` + } + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return "", fmt.Errorf("failed to decode client version response: %w", err) + } + + return result.Result, nil +} + +// GetLatestClientVersion fetches the client version for the latest available snapshot +func (s *SnapshotsAdapter) GetLatestClientVersion(ctx context.Context, network, client string) (string, error) { + blockNumber, err := s.GetLatestBlockNumber(ctx, network, client) + if err != nil { + return "", fmt.Errorf("failed to get latest block number: %w", err) + } + return s.GetClientVersion(ctx, network, client, blockNumber) +} diff --git a/internal/adapters/apis/tropidatooor/trpoidatooor_adapter.go b/internal/adapters/apis/tropidatooor/trpoidatooor_adapter.go index 1c289e2..87100e3 100644 --- a/internal/adapters/apis/tropidatooor/trpoidatooor_adapter.go +++ b/internal/adapters/apis/tropidatooor/trpoidatooor_adapter.go @@ -2,7 +2,6 @@ package tropidatooor import ( "clients-test/internal/application/domain" - "clients-test/internal/logger" "context" "encoding/json" "fmt" @@ -11,72 +10,59 @@ import ( // TropidatooorAdapter is the adapter to interact with the Tropidatooor API type TropidatooorAdapter struct { - baseURL string - client *http.Client - logPrefix string + baseURL string + client *http.Client } // NewTropidatooorAdapter creates a new TropidatooorAdapter func NewTropidatooorAdapter(baseURL string) *TropidatooorAdapter { return &TropidatooorAdapter{ - baseURL: baseURL, - client: &http.Client{}, - logPrefix: "TropidatooorAdapter", + baseURL: baseURL, + client: &http.Client{}, } } // DataRequest sends a request to the Tropidatooor API to request data for a specific backend func (t *TropidatooorAdapter) DataRequest(ctx context.Context, backendName string) (*domain.Mount, error) { - logger.DebugWithPrefix(t.logPrefix, "DataRequest: backendName=%s", backendName) req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("%s/data/request/%s", t.baseURL, backendName), nil) if err != nil { - logger.ErrorWithPrefix(t.logPrefix, "DataRequest: failed to create request: %v", err) return nil, fmt.Errorf("failed to create request: %w", err) } resp, err := t.client.Do(req) if err != nil { - logger.ErrorWithPrefix(t.logPrefix, "DataRequest: failed to send request: %v", err) return nil, fmt.Errorf("failed to send request: %w", err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - logger.ErrorWithPrefix(t.logPrefix, "DataRequest: non-200 status: %s", resp.Status) return nil, fmt.Errorf("failed to request data for %s: status %s", backendName, resp.Status) } var dataResponse domain.Mount if err := json.NewDecoder(resp.Body).Decode(&dataResponse); err != nil { - logger.ErrorWithPrefix(t.logPrefix, "DataRequest: failed to decode response: %v", err) return nil, fmt.Errorf("failed to decode response: %w", err) } - logger.DebugWithPrefix(t.logPrefix, "DataRequest: success, dataResponse=%+v", dataResponse) return &dataResponse, nil } // DataRelease sends a request to the Tropidatooor API to release data for a specific uniqueId func (t *TropidatooorAdapter) DataRelease(ctx context.Context, uniqueId string) error { - logger.DebugWithPrefix(t.logPrefix, "DataRelease: uniqueId=%s", uniqueId) req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("%s/data/release/%s", t.baseURL, uniqueId), nil) if err != nil { - logger.ErrorWithPrefix(t.logPrefix, "DataRelease: failed to create request: %v", err) return fmt.Errorf("failed to create request: %w", err) } resp, err := t.client.Do(req) if err != nil { - logger.ErrorWithPrefix(t.logPrefix, "DataRelease: failed to send request: %v", err) return fmt.Errorf("failed to send request: %w", err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - logger.ErrorWithPrefix(t.logPrefix, "DataRelease: non-200 status: %s", resp.Status) return fmt.Errorf("failed to release data for %s: status %s", uniqueId, resp.Status) } - logger.DebugWithPrefix(t.logPrefix, "DataRelease: success for uniqueId=%s", uniqueId) return nil } diff --git a/internal/adapters/composite/cleaner/cleaner.go b/internal/adapters/composite/cleaner/cleaner.go index b704e09..7d24116 100644 --- a/internal/adapters/composite/cleaner/cleaner.go +++ b/internal/adapters/composite/cleaner/cleaner.go @@ -6,59 +6,44 @@ import ( "clients-test/internal/adapters/apis/dappmanager" "clients-test/internal/adapters/apis/docker" "clients-test/internal/adapters/apis/execution" - "clients-test/internal/adapters/apis/tropidatooor" - "clients-test/internal/adapters/system/mount" "clients-test/internal/application/domain" "context" "fmt" ) type CleanerAdapter struct { - Dappmanager *dappmanager.DappManagerAdapter - Execution *execution.ExecutionAdapter - Brain *brain.BrainAdapter - Beaconchain *beaconchain.BeaconchainAdapter - Docker *docker.DockerAdapter - Mount *mount.MountAdapter - Tropidatooor *tropidatooor.TropidatooorAdapter + Dappmanager *dappmanager.DappManagerAdapter + Execution *execution.ExecutionAdapter + Brain *brain.BrainAdapter + Beaconchain *beaconchain.BeaconchainAdapter + Docker *docker.DockerAdapter } -func NewCleanerAdapter(dappmanager *dappmanager.DappManagerAdapter, execution *execution.ExecutionAdapter, brain *brain.BrainAdapter, beaconchain *beaconchain.BeaconchainAdapter, docker *docker.DockerAdapter, mount *mount.MountAdapter, tropidatooor *tropidatooor.TropidatooorAdapter) *CleanerAdapter { +func NewCleanerAdapter(dappmanager *dappmanager.DappManagerAdapter, execution *execution.ExecutionAdapter, brain *brain.BrainAdapter, beaconchain *beaconchain.BeaconchainAdapter, docker *docker.DockerAdapter) *CleanerAdapter { return &CleanerAdapter{ - Dappmanager: dappmanager, - Execution: execution, - Brain: brain, - Beaconchain: beaconchain, - Docker: docker, - Mount: mount, - Tropidatooor: tropidatooor, + Dappmanager: dappmanager, + Execution: execution, + Brain: brain, + Beaconchain: beaconchain, + Docker: docker, } } -// CleanEnvironment release the mounted volume and remove non-core packages -func (e *CleanerAdapter) CleanEnvironment(ctx context.Context, stakerConfig domain.StakerConfig, mountConfig domain.Mount) error { +// CleanEnvironment stops containers and removes non-core packages +func (e *CleanerAdapter) CleanEnvironment(ctx context.Context, stakerConfig domain.StakerConfig) error { var errs []error - // Attempt to stop container and release mounted volume - volumeTarget, err := e.Docker.StopAndGetVolumeTarget(ctx, stakerConfig.ExecutionContainerName, stakerConfig.ExecutionVolumeName) - if err != nil { - errs = append(errs, fmt.Errorf("stop container failed: %w", err)) - } else { - if err := e.Mount.UnmountNFS(ctx, volumeTarget); err != nil { - errs = append(errs, fmt.Errorf("failed to unmount NFS: %w", err)) - } - } - - // Attempt to release data - if err := e.Tropidatooor.DataRelease(ctx, mountConfig.Id); err != nil { - errs = append(errs, fmt.Errorf("failed to release data for uniqueId %s: %w", mountConfig.Id, err)) - } + // Attempt to stop container + // _, err := e.Docker.StopAndGetVolumeTarget(ctx, stakerConfig.ExecutionContainerName, stakerConfig.ExecutionVolumeName) + // if err != nil { + // errs = append(errs, fmt.Errorf("stop container failed: %w", err)) + // } // Attempt to remove non-core packages - pkgErrs := e.Dappmanager.RemoveNonCorePackages(ctx) - for _, pkgErr := range pkgErrs { - errs = append(errs, fmt.Errorf("remove non-core package failed: %w", pkgErr)) - } + // pkgErrs := e.Dappmanager.RemoveNonCorePackages(ctx) + // for _, pkgErr := range pkgErrs { + // errs = append(errs, fmt.Errorf("remove non-core package failed: %w", pkgErr)) + // } // Return combined error if any step failed if len(errs) > 0 { diff --git a/internal/adapters/composite/composite.go b/internal/adapters/composite/composite.go index cfb46be..eca6f3c 100644 --- a/internal/adapters/composite/composite.go +++ b/internal/adapters/composite/composite.go @@ -6,46 +6,98 @@ import ( "clients-test/internal/adapters/apis/dappmanager" "clients-test/internal/adapters/apis/docker" "clients-test/internal/adapters/apis/execution" + "clients-test/internal/adapters/apis/github" "clients-test/internal/adapters/apis/ipfs" - "clients-test/internal/adapters/apis/tropidatooor" + "clients-test/internal/adapters/apis/snapshots" "clients-test/internal/adapters/composite/cleaner" "clients-test/internal/adapters/composite/ensurer" "clients-test/internal/adapters/composite/executor" - "clients-test/internal/adapters/system/mount" + "clients-test/internal/adapters/shared/blocknumber" "clients-test/internal/application/domain" "context" + "time" ) type CompositeAdapter struct { ensurer *ensurer.EnsurerAdapter executor *executor.ExecutorAdapter cleaner *cleaner.CleanerAdapter + docker *docker.DockerAdapter + github *github.GitHubAdapter + report *domain.TestReport } func NewCompositeAdapter( dappManagerAdapter *dappmanager.DappManagerAdapter, brainAdapter *brain.BrainAdapter, - tropidatooorAdapter *tropidatooor.TropidatooorAdapter, dockerAdapter *docker.DockerAdapter, - mountAdapter *mount.MountAdapter, + snapshotsAdapter *snapshots.SnapshotsAdapter, beaconchainAdapter *beaconchain.BeaconchainAdapter, executionAdapter *execution.ExecutionAdapter, ipfsAdapter *ipfs.IPFSAdapter, + githubAdapter *github.GitHubAdapter, + blockNumberAdapter *blocknumber.BlockNumberAdapter, ) *CompositeAdapter { - ensurer := ensurer.NewEnsurerAdapter(dappManagerAdapter, brainAdapter, tropidatooorAdapter, dockerAdapter, mountAdapter, beaconchainAdapter, executionAdapter, ipfsAdapter) + ensurer := ensurer.NewEnsurerAdapter(dappManagerAdapter, brainAdapter, dockerAdapter, snapshotsAdapter, beaconchainAdapter, executionAdapter, ipfsAdapter, blockNumberAdapter) executor := executor.NewExecutorAdapter(executionAdapter, brainAdapter, beaconchainAdapter) - cleaner := cleaner.NewCleanerAdapter(dappManagerAdapter, executionAdapter, brainAdapter, beaconchainAdapter, dockerAdapter, mountAdapter, tropidatooorAdapter) - return &CompositeAdapter{ensurer, executor, cleaner} + cleaner := cleaner.NewCleanerAdapter(dappManagerAdapter, executionAdapter, brainAdapter, beaconchainAdapter, dockerAdapter) + return &CompositeAdapter{ + ensurer: ensurer, + executor: executor, + cleaner: cleaner, + docker: dockerAdapter, + github: githubAdapter, + } } -func (t *CompositeAdapter) EnsureEnvironment(ctx context.Context, mountConfig domain.Mount, stakerConfig domain.StakerConfig, pkg domain.Pkg) error { - return t.ensurer.EnsureEnvironment(ctx, mountConfig, stakerConfig, pkg) +func (t *CompositeAdapter) EnsureEnvironment(ctx context.Context, stakerConfig domain.StakerConfig, pkg domain.Pkg) error { + // Initialize the report + t.report = domain.NewTestReport(stakerConfig) + + return t.ensurer.EnsureEnvironment(ctx, stakerConfig, pkg, t.report) +} + +func (t *CompositeAdapter) ExecuteTest(ctx context.Context, stakerConfig domain.StakerConfig) error { + // Record test start time for log collection + testStartTime := time.Now() + + // Run the actual test + testErr := t.executor.ExecuteTest(ctx, t.report) + + // Record test end time + testEndTime := time.Now() + + // Collect container error logs from all relevant containers + t.collectContainerErrorLogs(ctx, stakerConfig, testStartTime, testEndTime) + + // Set the final result + t.report.SetResult(testErr == nil, testErr) + + // Comment on PR if GitHub integration is enabled (ignore errors - don't fail test for PR comment issues) + _ = t.github.CommentOnPR(ctx, t.report) + + return testErr } -func (t *CompositeAdapter) ExecuteTest(ctx context.Context) error { - return t.executor.ExecuteTest(ctx) +func (t *CompositeAdapter) CleanEnvironment(ctx context.Context, stakerConfig domain.StakerConfig) error { + return t.cleaner.CleanEnvironment(ctx, stakerConfig) } -func (t *CompositeAdapter) CleanEnvironment(ctx context.Context, stakerConfig domain.StakerConfig, mountConfig domain.Mount) error { - return t.cleaner.CleanEnvironment(ctx, stakerConfig, mountConfig) +// collectContainerErrorLogs collects error logs from all relevant containers +func (t *CompositeAdapter) collectContainerErrorLogs(ctx context.Context, stakerConfig domain.StakerConfig, since, until time.Time) { + const maxLinesPerContainer = 3 + + containerNames := []string{ + stakerConfig.BrainContainerName, + stakerConfig.SignerContainerName, + stakerConfig.BeaconchainContainerName, + stakerConfig.ValidatorContainerName, + stakerConfig.ExecutionContainerName, + } + + errorLogs := t.docker.CollectAllContainerErrorLogs(ctx, containerNames, since, until, maxLinesPerContainer) + + for containerName, lines := range errorLogs { + t.report.AddContainerErrors(containerName, lines) + } } diff --git a/internal/adapters/composite/ensurer/ensurer.go b/internal/adapters/composite/ensurer/ensurer.go index aadf4f0..255e023 100644 --- a/internal/adapters/composite/ensurer/ensurer.go +++ b/internal/adapters/composite/ensurer/ensurer.go @@ -3,6 +3,8 @@ package ensurer import ( "context" "fmt" + "strconv" + "time" "clients-test/internal/adapters/apis/beaconchain" "clients-test/internal/adapters/apis/brain" @@ -10,54 +12,111 @@ import ( "clients-test/internal/adapters/apis/docker" "clients-test/internal/adapters/apis/execution" "clients-test/internal/adapters/apis/ipfs" - "clients-test/internal/adapters/apis/tropidatooor" - "clients-test/internal/adapters/system/mount" + "clients-test/internal/adapters/apis/snapshots" + "clients-test/internal/adapters/shared/blocknumber" "clients-test/internal/application/domain" ) type EnsurerAdapter struct { - DappManager *dappmanager.DappManagerAdapter - Brain *brain.BrainAdapter - Tropidatooor *tropidatooor.TropidatooorAdapter - Docker *docker.DockerAdapter - Mount *mount.MountAdapter - Beaconchain *beaconchain.BeaconchainAdapter - Execution *execution.ExecutionAdapter - Ipfs *ipfs.IPFSAdapter + DappManager *dappmanager.DappManagerAdapter + Brain *brain.BrainAdapter + Docker *docker.DockerAdapter + Snapshots *snapshots.SnapshotsAdapter + Beaconchain *beaconchain.BeaconchainAdapter + Execution *execution.ExecutionAdapter + Ipfs *ipfs.IPFSAdapter + BlockNumber *blocknumber.BlockNumberAdapter } -func NewEnsurerAdapter(dappManager *dappmanager.DappManagerAdapter, brain *brain.BrainAdapter, tropidatooor *tropidatooor.TropidatooorAdapter, docker *docker.DockerAdapter, mountAdapter *mount.MountAdapter, beaconchain *beaconchain.BeaconchainAdapter, execution *execution.ExecutionAdapter, ipfs *ipfs.IPFSAdapter) *EnsurerAdapter { +func NewEnsurerAdapter(dappManager *dappmanager.DappManagerAdapter, brain *brain.BrainAdapter, docker *docker.DockerAdapter, snapshotsAdapter *snapshots.SnapshotsAdapter, beaconchain *beaconchain.BeaconchainAdapter, execution *execution.ExecutionAdapter, ipfs *ipfs.IPFSAdapter, blockNumberAdapter *blocknumber.BlockNumberAdapter) *EnsurerAdapter { return &EnsurerAdapter{ - DappManager: dappManager, - Brain: brain, - Tropidatooor: tropidatooor, - Docker: docker, - Mount: mountAdapter, - Beaconchain: beaconchain, - Execution: execution, - Ipfs: ipfs, + DappManager: dappManager, + Brain: brain, + Docker: docker, + Snapshots: snapshotsAdapter, + Beaconchain: beaconchain, + Execution: execution, + Ipfs: ipfs, + BlockNumber: blockNumberAdapter, } } +// timeOperation measures the duration of an operation and records it in the report +func timeOperation(report *domain.TestReport, operationName string, fn func() error) error { + start := time.Now() + err := fn() + duration := time.Since(start) + + success := err == nil + report.AddEnsureTiming(operationName, duration, success, err) + + return err +} + // EnsureEnvironment validates the environment and prepares it for testing. -// It sets the staker config, installs the package, stops the container, mounts the NFS, and starts the container. -func (e *EnsurerAdapter) EnsureEnvironment(ctx context.Context, mountConfig domain.Mount, stakerConfig domain.StakerConfig, pkg domain.Pkg) error { - if err := e.DappManager.SetStakerConfig(ctx, stakerConfig); err != nil { - return fmt.Errorf("failed to set staker config for DNP: %w", err) +// It sets the staker config and installs the package. +// All operations are timed and recorded in the report. +func (e *EnsurerAdapter) EnsureEnvironment(ctx context.Context, stakerConfig domain.StakerConfig, pkg domain.Pkg, report *domain.TestReport) error { + // Determine what type of client is being tested + isExecutionTest := pkg.DnpName == stakerConfig.ExecutionDnpName + isConsensusTest := pkg.DnpName == stakerConfig.ConsensusDnpName + + if isExecutionTest { + report.TestedClientType = "execution" + } else if isConsensusTest { + report.TestedClientType = "consensus" } - if err := e.DappManager.PackageInstall(ctx, pkg); err != nil { - return fmt.Errorf("failed to install package: %w", err) + + // Read snapshot block number + e.readSnapshotBlockNumber(ctx, report) + + // SetStakerConfig + if err := timeOperation(report, "SetStakerConfig", func() error { + return e.DappManager.SetStakerConfig(ctx, stakerConfig) + }); err != nil { + return fmt.Errorf("failed to set staker config for DNP: %w", err) } - volumeTarget, err := e.Docker.StopAndGetVolumeTarget(ctx, stakerConfig.ExecutionContainerName, stakerConfig.ExecutionVolumeName) - if err != nil { - return fmt.Errorf("failed to stop container and get volume: %w", err) + + // Capture client version BEFORE install (if client is already running) + if isExecutionTest { + if version, err := e.Execution.GetClientVersion(ctx); err == nil { + report.ExecutionClientVersionBefore = version + } + } else if isConsensusTest { + if version, err := e.Beaconchain.GetClientVersion(ctx); err == nil { + report.ConsensusClientVersionBefore = version + } } - if err := e.Mount.MountNFS(ctx, mountConfig.Path, volumeTarget); err != nil { - return fmt.Errorf("failed to mount NFS: %w", err) + // PackageInstall + if err := timeOperation(report, "PackageInstall", func() error { + return e.DappManager.PackageInstall(ctx, pkg) + }); err != nil { + return fmt.Errorf("failed to install package: %w", err) } - if err := e.Docker.StartContainer(ctx, stakerConfig.ExecutionContainerName); err != nil { - return fmt.Errorf("failed to start container: %w", err) + + // Capture client version AFTER install + if isExecutionTest { + if version, err := e.Execution.GetClientVersion(ctx); err == nil { + report.ExecutionClientVersionAfter = version + } + } else if isConsensusTest { + if version, err := e.Beaconchain.GetClientVersion(ctx); err == nil { + report.ConsensusClientVersionAfter = version + } } + return nil } + +// readSnapshotBlockNumber reads the snapshot block number from the execution client's volume +// and stores it in the report. Errors are silently ignored as this is informational. +func (e *EnsurerAdapter) readSnapshotBlockNumber(ctx context.Context, report *domain.TestReport) { + // Use the BlockNumber adapter which is already initialized with the correct path + blockNumberStr, err := e.BlockNumber.ReadBlockNumber(ctx) + if err == nil && blockNumberStr != "" { + if blockNumber, parseErr := strconv.ParseUint(blockNumberStr, 10, 64); parseErr == nil { + report.SnapshotBlockNumber = blockNumber + } + } +} diff --git a/internal/adapters/composite/executor/executor.go b/internal/adapters/composite/executor/executor.go index 746f561..6e41ef9 100644 --- a/internal/adapters/composite/executor/executor.go +++ b/internal/adapters/composite/executor/executor.go @@ -8,6 +8,7 @@ import ( "clients-test/internal/adapters/apis/beaconchain" "clients-test/internal/adapters/apis/brain" "clients-test/internal/adapters/apis/execution" + "clients-test/internal/application/domain" ) type ExecutorAdapter struct { @@ -24,11 +25,23 @@ func NewExecutorAdapter(execution *execution.ExecutionAdapter, brain *brain.Brai } } +// timeOperation measures the duration of an operation and records it in the report +func timeOperation(report *domain.TestReport, operationName string, fn func() error) error { + start := time.Now() + err := fn() + duration := time.Since(start) + + success := err == nil + report.AddExecuteTiming(operationName, duration, success, err) + + return err +} + // waitForExecutionSync waits until the execution client is synced or times out, // returning only after maxTries with the most recent error (if any). func (t *ExecutorAdapter) waitForExecutionSync(ctx context.Context) error { const ( - maxTries = 60 + maxTries = 180 sleepDur = 6 * time.Second ) var lastErr error @@ -162,15 +175,25 @@ func (t *ExecutorAdapter) waitForValidatorLiveness(ctx context.Context) error { } // ExecuteTest runs both sync and liveness checks in sequence -func (t *ExecutorAdapter) ExecuteTest(ctx context.Context) error { - if err := t.waitForBeaconchainSync(ctx); err != nil { +// All operations are timed and recorded in the report. +func (t *ExecutorAdapter) ExecuteTest(ctx context.Context, report *domain.TestReport) error { + if err := timeOperation(report, "WaitForBeaconchainSync", func() error { + return t.waitForBeaconchainSync(ctx) + }); err != nil { return err } - if err := t.waitForExecutionSync(ctx); err != nil { + + if err := timeOperation(report, "WaitForExecutionSync", func() error { + return t.waitForExecutionSync(ctx) + }); err != nil { return err } - if err := t.waitForValidatorLiveness(ctx); err != nil { + + if err := timeOperation(report, "WaitForValidatorLiveness", func() error { + return t.waitForValidatorLiveness(ctx) + }); err != nil { return err } + return nil } diff --git a/internal/adapters/composite/snapshotmanager/snapshotmanager.go b/internal/adapters/composite/snapshotmanager/snapshotmanager.go new file mode 100644 index 0000000..c192831 --- /dev/null +++ b/internal/adapters/composite/snapshotmanager/snapshotmanager.go @@ -0,0 +1,59 @@ +package snapshotmanager + +import ( + "context" + "fmt" + + "clients-test/internal/adapters/apis/docker" + "clients-test/internal/adapters/apis/snapshots" + "clients-test/internal/application/domain" +) + +const ( + downloadContainerPrefix = "snapshot-download-" +) + +// SnapshotManagerAdapter is a composite adapter that combines snapshots and docker adapters +// to provide high-level snapshot management operations +type SnapshotManagerAdapter struct { + snapshots *snapshots.SnapshotsAdapter + docker *docker.DockerAdapter +} + +// NewSnapshotManagerAdapter creates a new SnapshotManagerAdapter +func NewSnapshotManagerAdapter( + snapshotsAdapter *snapshots.SnapshotsAdapter, + dockerAdapter *docker.DockerAdapter, +) *SnapshotManagerAdapter { + return &SnapshotManagerAdapter{ + snapshots: snapshotsAdapter, + docker: dockerAdapter, + } +} + +// DownloadAndMountSnapshot performs the complete snapshot download and mount process: +// 1. Stops the client container (if running) +// 2. Downloads and extracts the snapshot to the volume +func (s *SnapshotManagerAdapter) DownloadAndMountSnapshot(ctx context.Context, network string, client domain.ExecutionClientInfo) error { + // 1. Stop the container (if running) - ignore errors as container might not exist + _ = s.docker.StopContainer(ctx, client.ContainerName) + + // 2. Download and extract snapshot to volume using Docker container + containerName := fmt.Sprintf("%s%s", downloadContainerPrefix, client.ShortName) + if err := s.docker.RunSnapshotDownload(ctx, containerName, client.ShortName, network, client.VolumeTargetPath, s.snapshots.GetBaseURL()); err != nil { + return fmt.Errorf("failed to download and extract snapshot: %w", err) + } + + return nil +} + +// GetLatestBlockNumber fetches the latest available block number for a client +func (s *SnapshotManagerAdapter) GetLatestBlockNumber(ctx context.Context, network, client string) (string, error) { + return s.snapshots.GetLatestBlockNumber(ctx, network, client) +} + +// StopDownload stops the snapshot download container for a specific client +func (s *SnapshotManagerAdapter) StopDownload(ctx context.Context, clientShortName string) { + containerName := fmt.Sprintf("%s%s", downloadContainerPrefix, clientShortName) + _ = s.docker.StopContainerWithTimeout(ctx, containerName, 5) +} diff --git a/internal/adapters/shared/blocknumber/blocknumber_adapter.go b/internal/adapters/shared/blocknumber/blocknumber_adapter.go new file mode 100644 index 0000000..f090f2c --- /dev/null +++ b/internal/adapters/shared/blocknumber/blocknumber_adapter.go @@ -0,0 +1,156 @@ +package blocknumber + +import ( + "clients-test/internal/application/domain" + "context" + "fmt" + "os" + "path/filepath" + "strconv" + "strings" + "syscall" +) + +// BlockNumberAdapter handles the snapshot_block_number file operations +type BlockNumberAdapter struct { + basePath string +} + +// NewBlockNumberAdapter creates a new BlockNumberAdapter with default path +func NewBlockNumberAdapter() *BlockNumberAdapter { + return &BlockNumberAdapter{ + basePath: domain.SnapshotProgressPath, + } +} + +// NewBlockNumberAdapterWithPath creates a new BlockNumberAdapter with a custom base path +func NewBlockNumberAdapterWithPath(basePath string) *BlockNumberAdapter { + return &BlockNumberAdapter{ + basePath: basePath, + } +} + +// blockNumberFilePath returns the full path to the block number file +func (b *BlockNumberAdapter) blockNumberFilePath() string { + return filepath.Join(b.basePath, domain.SnapshotBlockNumberFileName) +} + +// WriteBlockNumber writes the block number to the snapshot_block_number file +func (b *BlockNumberAdapter) WriteBlockNumber(ctx context.Context, blockNumber string) error { + filePath := b.blockNumberFilePath() + + // Ensure directory exists + if err := os.MkdirAll(b.basePath, 0755); err != nil { + return fmt.Errorf("failed to create directory %s: %w", b.basePath, err) + } + + f, err := os.Create(filePath) + if err != nil { + return fmt.Errorf("failed to create block number file: %w", err) + } + defer f.Close() + + // Acquire exclusive lock for writing + if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX); err != nil { + return fmt.Errorf("failed to acquire lock on block number file: %w", err) + } + defer syscall.Flock(int(f.Fd()), syscall.LOCK_UN) + + _, err = f.WriteString(blockNumber) + if err != nil { + return fmt.Errorf("failed to write to block number file: %w", err) + } + + return nil +} + +// ReadBlockNumber reads the block number from the snapshot_block_number file +// Returns empty string if file doesn't exist +func (b *BlockNumberAdapter) ReadBlockNumber(ctx context.Context) (string, error) { + filePath := b.blockNumberFilePath() + + f, err := os.Open(filePath) + if err != nil { + if os.IsNotExist(err) { + return "", nil + } + return "", fmt.Errorf("failed to open block number file: %w", err) + } + defer f.Close() + + // Acquire shared lock for reading + if err := syscall.Flock(int(f.Fd()), syscall.LOCK_SH); err != nil { + return "", fmt.Errorf("failed to acquire lock on block number file: %w", err) + } + defer syscall.Flock(int(f.Fd()), syscall.LOCK_UN) + + data, err := os.ReadFile(filePath) + if err != nil { + return "", fmt.Errorf("failed to read block number file: %w", err) + } + + blockNumber := strings.TrimSpace(string(data)) + return blockNumber, nil +} + +// BlockNumberExists checks if a block number file exists +func (b *BlockNumberAdapter) BlockNumberExists(ctx context.Context) (bool, error) { + filePath := b.blockNumberFilePath() + + f, err := os.Open(filePath) + if err != nil { + if os.IsNotExist(err) { + return false, nil + } + return false, fmt.Errorf("failed to open block number file: %w", err) + } + defer f.Close() + + // Acquire shared lock for reading + if err := syscall.Flock(int(f.Fd()), syscall.LOCK_SH); err != nil { + return false, fmt.Errorf("failed to acquire lock on block number file: %w", err) + } + defer syscall.Flock(int(f.Fd()), syscall.LOCK_UN) + + return true, nil +} + +// CompareBlockNumbers compares two block numbers +// Returns: -1 if a < b, 0 if a == b, 1 if a > b +func (b *BlockNumberAdapter) CompareBlockNumbers(a, blockB string) int { + aInt, aErr := strconv.ParseInt(a, 10, 64) + bInt, bErr := strconv.ParseInt(blockB, 10, 64) + + if aErr != nil || bErr != nil { + // Fall back to string comparison if not valid integers + if a < blockB { + return -1 + } else if a > blockB { + return 1 + } + return 0 + } + + if aInt < bInt { + return -1 + } else if aInt > bInt { + return 1 + } + return 0 +} + +// IsNewerSnapshot checks if the latest available block number is newer than the current one +func (b *BlockNumberAdapter) IsNewerSnapshot(ctx context.Context, latestBlockNumber string) (bool, error) { + currentBlockNumber, err := b.ReadBlockNumber(ctx) + if err != nil { + return false, err + } + + // If no current block number, we need to download + if currentBlockNumber == "" { + return true, nil + } + + // Compare block numbers + return b.CompareBlockNumbers(latestBlockNumber, currentBlockNumber) > 0, nil +} diff --git a/internal/adapters/shared/download/download_adapter.go b/internal/adapters/shared/download/download_adapter.go new file mode 100644 index 0000000..a141427 --- /dev/null +++ b/internal/adapters/shared/download/download_adapter.go @@ -0,0 +1,119 @@ +package download + +import ( + "clients-test/internal/application/domain" + "context" + "fmt" + "os" + "path/filepath" + "syscall" + "time" +) + +// DownloadAdapter handles the .download_in_progress file operations +type DownloadAdapter struct { + basePath string +} + +// NewDownloadAdapter creates a new ProgressAdapter +func NewDownloadAdapter() *DownloadAdapter { + return &DownloadAdapter{ + basePath: domain.SnapshotProgressPath, + } +} + +// NewDownloadAdapterWithPath creates a new DownloadAdapter with a custom base path (for testing) +func NewDownloadAdapterWithPath(basePath string) *DownloadAdapter { + return &DownloadAdapter{ + basePath: basePath, + } +} + +// progressFilePath returns the full path to the progress file +func (p *DownloadAdapter) progressFilePath() string { + return filepath.Join(p.basePath, domain.ProgressFileName) +} + +// SetDownloadInProgress creates the .download_in_progress file +func (p *DownloadAdapter) SetDownloadInProgress(ctx context.Context) error { + filePath := p.progressFilePath() + + if err := os.MkdirAll(p.basePath, 0755); err != nil { + return fmt.Errorf("failed to create directory %s: %w", p.basePath, err) + } + + f, err := os.Create(filePath) + if err != nil { + return fmt.Errorf("failed to create progress file: %w", err) + } + defer f.Close() + + // Acquire exclusive lock for writing + if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX); err != nil { + return fmt.Errorf("failed to acquire lock on progress file: %w", err) + } + defer syscall.Flock(int(f.Fd()), syscall.LOCK_UN) + + _, err = f.WriteString(time.Now().UTC().Format(time.RFC3339)) + if err != nil { + return fmt.Errorf("failed to write to progress file: %w", err) + } + + return nil +} + +// ClearDownloadInProgress removes the .download_in_progress file +func (p *DownloadAdapter) ClearDownloadInProgress(ctx context.Context) error { + filePath := p.progressFilePath() + + err := os.Remove(filePath) + if err != nil && !os.IsNotExist(err) { + return fmt.Errorf("failed to remove progress file: %w", err) + } + + return nil +} + +// IsDownloadInProgress checks if download is in progress +func (p *DownloadAdapter) IsDownloadInProgress(ctx context.Context) (bool, error) { + filePath := p.progressFilePath() + + f, err := os.Open(filePath) + if err != nil { + if os.IsNotExist(err) { + return false, nil + } + return false, fmt.Errorf("failed to open progress file: %w", err) + } + defer f.Close() + + // Acquire shared lock for reading + if err := syscall.Flock(int(f.Fd()), syscall.LOCK_SH); err != nil { + return false, fmt.Errorf("failed to acquire lock on progress file: %w", err) + } + defer syscall.Flock(int(f.Fd()), syscall.LOCK_UN) + + return true, nil +} + +// WaitForDownloadComplete waits until no download is in progress +// Returns an error if the context is cancelled or times out +func (p *DownloadAdapter) WaitForDownloadComplete(ctx context.Context, checkInterval time.Duration) error { + for { + select { + case <-ctx.Done(): + return ctx.Err() + default: + inProgress, err := p.IsDownloadInProgress(ctx) + if err != nil { + return fmt.Errorf("error checking download progress: %w", err) + } + + if !inProgress { + return nil + } + + time.Sleep(checkInterval) + } + } +} diff --git a/internal/adapters/shared/testing/testing_adapter.go b/internal/adapters/shared/testing/testing_adapter.go new file mode 100644 index 0000000..121cba4 --- /dev/null +++ b/internal/adapters/shared/testing/testing_adapter.go @@ -0,0 +1,119 @@ +package testing + +import ( + "clients-test/internal/application/domain" + "context" + "fmt" + "os" + "path/filepath" + "syscall" + "time" +) + +// TestAdapter handles the .test_in_progress file operations +type TestAdapter struct { + basePath string +} + +// NewTestAdapter creates a new TestAdapter +func NewTestAdapter() *TestAdapter { + return &TestAdapter{ + basePath: domain.SnapshotProgressPath, + } +} + +// NewTestAdapterWithPath creates a new TestAdapter with a custom base path (for testing) +func NewTestAdapterWithPath(basePath string) *TestAdapter { + return &TestAdapter{ + basePath: basePath, + } +} + +// progressFilePath returns the full path to the test progress file +func (t *TestAdapter) progressFilePath() string { + return filepath.Join(t.basePath, domain.TestProgressFileName) +} + +// SetTestInProgress creates the .test_in_progress file +func (t *TestAdapter) SetTestInProgress(ctx context.Context) error { + filePath := t.progressFilePath() + + if err := os.MkdirAll(t.basePath, 0755); err != nil { + return fmt.Errorf("failed to create directory %s: %w", t.basePath, err) + } + + f, err := os.Create(filePath) + if err != nil { + return fmt.Errorf("failed to create test progress file: %w", err) + } + defer f.Close() + + // Acquire exclusive lock for writing + if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX); err != nil { + return fmt.Errorf("failed to acquire lock on test progress file: %w", err) + } + defer syscall.Flock(int(f.Fd()), syscall.LOCK_UN) + + _, err = f.WriteString(time.Now().UTC().Format(time.RFC3339)) + if err != nil { + return fmt.Errorf("failed to write to test progress file: %w", err) + } + + return nil +} + +// ClearTestInProgress removes the .test_in_progress file +func (t *TestAdapter) ClearTestInProgress(ctx context.Context) error { + filePath := t.progressFilePath() + + err := os.Remove(filePath) + if err != nil && !os.IsNotExist(err) { + return fmt.Errorf("failed to remove test progress file: %w", err) + } + + return nil +} + +// IsTestInProgress checks if a test is in progress +func (t *TestAdapter) IsTestInProgress(ctx context.Context) (bool, error) { + filePath := t.progressFilePath() + + f, err := os.Open(filePath) + if err != nil { + if os.IsNotExist(err) { + return false, nil + } + return false, fmt.Errorf("failed to open test progress file: %w", err) + } + defer f.Close() + + // Acquire shared lock for reading + if err := syscall.Flock(int(f.Fd()), syscall.LOCK_SH); err != nil { + return false, fmt.Errorf("failed to acquire lock on test progress file: %w", err) + } + defer syscall.Flock(int(f.Fd()), syscall.LOCK_UN) + + return true, nil +} + +// WaitForTestComplete waits until no test is in progress +// Returns an error if the context is cancelled or times out +func (t *TestAdapter) WaitForTestComplete(ctx context.Context, checkInterval time.Duration) error { + for { + select { + case <-ctx.Done(): + return ctx.Err() + default: + inProgress, err := t.IsTestInProgress(ctx) + if err != nil { + return fmt.Errorf("error checking test progress: %w", err) + } + + if !inProgress { + return nil + } + + time.Sleep(checkInterval) + } + } +} diff --git a/internal/adapters/system/mount/mount_adapter.go b/internal/adapters/system/mount/mount_adapter.go index f90f85d..7993dbe 100644 --- a/internal/adapters/system/mount/mount_adapter.go +++ b/internal/adapters/system/mount/mount_adapter.go @@ -4,40 +4,30 @@ import ( "context" "fmt" "os/exec" - - "clients-test/internal/logger" ) -type MountAdapter struct { - logPrefix string -} +type MountAdapter struct{} func NewMountAdapter() *MountAdapter { - return &MountAdapter{logPrefix: "MountAdapter"} + return &MountAdapter{} } // MountNFS mounts the NFS share at srcPath to the targetPath using sudo mount -t nfs func (m *MountAdapter) MountNFS(ctx context.Context, srcPath, targetPath string) error { - logger.DebugWithPrefix(m.logPrefix, "MountNFS called with srcPath=%s, targetPath=%s", srcPath, targetPath) mountCmd := exec.CommandContext(ctx, "sudo", "mount", "-t", "nfs", srcPath, targetPath) output, err := mountCmd.CombinedOutput() if err != nil { - logger.ErrorWithPrefix(m.logPrefix, "Failed to mount %s onto %s: %v\n%s", srcPath, targetPath, err, string(output)) return fmt.Errorf("failed to mount %s onto %s: %v\n%s", srcPath, targetPath, err, string(output)) } - logger.DebugWithPrefix(m.logPrefix, "Successfully mounted %s onto %s", srcPath, targetPath) return nil } // UnmountNFS unmounts the filesystem at targetPath using sudo umount func (m *MountAdapter) UnmountNFS(ctx context.Context, targetPath string) error { - logger.DebugWithPrefix(m.logPrefix, "UnmountNFS called with targetPath=%s", targetPath) umountCmd := exec.CommandContext(ctx, "sudo", "umount", targetPath) output, err := umountCmd.CombinedOutput() if err != nil { - logger.ErrorWithPrefix(m.logPrefix, "Failed to unmount %s: %v\n%s", targetPath, err, string(output)) return fmt.Errorf("failed to unmount %s: %v\n%s", targetPath, err, string(output)) } - logger.DebugWithPrefix(m.logPrefix, "Successfully unmounted %s", targetPath) return nil } diff --git a/internal/application/domain/docker_names.go b/internal/application/domain/docker_names.go new file mode 100644 index 0000000..d7f30d2 --- /dev/null +++ b/internal/application/domain/docker_names.go @@ -0,0 +1,44 @@ +package domain + +import ( + "fmt" + "strings" +) + +// Utility to get the servicename from a execution client (this utility assumes the service name is the first part of the dnp name) +// e.g hoodi-nethermind.dnp.dappnode.eth -> nethermind +// e.g nethermind-hoodi.dnp.dappnode.eth -> nethermind +// e.g nethermind-hoodi.public.dappnode.eth -> nethermind +// e.g geth.dnp.dappnode.eth -> geth +// e.g reth-gnosis.dnp.dappnode.eth -> reth +// e.g gnosis-reth.dnp.dappnode.eth -> reth +func serviceNameFromExecutionClient(dnpName, network string) string { + trimmed := strings.TrimSuffix(dnpName, ".dnp.dappnode.eth") + trimmed = strings.TrimSuffix(trimmed, ".public.dappnode.eth") + parts := strings.Split(trimmed, "-") + + // If there is only one part, return it + if len(parts) == 1 { + return parts[0] + } + // If there are multiple parts, find and remove the network part + for i, part := range parts { + if part == network { + parts = append(parts[:i], parts[i+1:]...) + break + } + } + // Return the remaining parts joined by "-" + return strings.Join(parts, "-") +} + +// Utility to get the container name from service and dnpName. append dnp or public suffix depending on original dnpName +func containerName(serviceName, dnpName string) string { + return fmt.Sprintf("DAppNodePackage-%s.%s", serviceName, dnpName) +} + +// Utility to get the docker volume name from dnpName and compose volume name +// i.e hoodi-nethermind.dnp.dappnode.eth -> hoodi-netherminddnpdappnodeeth_ +func composeVolumeName(dnpName, composeVolumeName string) string { + return fmt.Sprintf("%s_%s", strings.ReplaceAll(dnpName, ".", ""), composeVolumeName) +} diff --git a/internal/application/domain/report.go b/internal/application/domain/report.go new file mode 100644 index 0000000..6f241f3 --- /dev/null +++ b/internal/application/domain/report.go @@ -0,0 +1,345 @@ +package domain + +import ( + "fmt" + "strings" + "time" +) + +// TimingEntry represents a single timed operation +type TimingEntry struct { + Operation string `json:"operation"` + Duration time.Duration `json:"duration"` + Success bool `json:"success"` + Error string `json:"error,omitempty"` +} + +// ContainerErrorLog represents error logs from a container +type ContainerErrorLog struct { + ContainerName string `json:"containerName"` + ErrorLines []string `json:"errorLines"` +} + +// TestReport holds all information for the test report +type TestReport struct { + // Client configuration + ExecutionDnpName string `json:"executionDnpName"` + ConsensusDnpName string `json:"consensusDnpName"` + Web3SignerDnpName string `json:"web3signerDnpName"` + MevBoostDnpName string `json:"mevBoostDnpName"` + Network string `json:"network"` + + // Client versions (before/after install) + ExecutionClientVersionBefore string `json:"executionClientVersionBefore,omitempty"` + ExecutionClientVersionAfter string `json:"executionClientVersionAfter,omitempty"` + ConsensusClientVersionBefore string `json:"consensusClientVersionBefore,omitempty"` + ConsensusClientVersionAfter string `json:"consensusClientVersionAfter,omitempty"` + SnapshotClientVersion string `json:"snapshotClientVersion,omitempty"` + + // blocknumber of the snapshot used + SnapshotBlockNumber uint64 `json:"snapshotBlockNumber,omitempty"` + + // What type of client is being tested + TestedClientType string `json:"testedClientType,omitempty"` // "execution" or "consensus" + + // Test execution info + StartTime time.Time `json:"startTime"` + EndTime time.Time `json:"endTime"` + + // Timing measurements + EnsureTimings []TimingEntry `json:"ensureTimings"` + ExecuteTimings []TimingEntry `json:"executeTimings"` + + // Container error logs + ContainerErrors []ContainerErrorLog `json:"containerErrors"` + + // Final result + Success bool `json:"success"` + ErrorMessage string `json:"errorMessage,omitempty"` +} + +// NewTestReport creates a new TestReport from StakerConfig +func NewTestReport(config StakerConfig) *TestReport { + return &TestReport{ + ExecutionDnpName: config.ExecutionDnpName, + ConsensusDnpName: config.ConsensusDnpName, + Web3SignerDnpName: config.Web3SignerDnpName, + MevBoostDnpName: config.MevBoostDnpName, + Network: config.Network, + StartTime: time.Now(), + EnsureTimings: make([]TimingEntry, 0), + ExecuteTimings: make([]TimingEntry, 0), + ContainerErrors: make([]ContainerErrorLog, 0), + Success: true, + } +} + +// AddEnsureTiming adds a timing entry for an ensure operation +func (r *TestReport) AddEnsureTiming(operation string, duration time.Duration, success bool, err error) { + entry := TimingEntry{ + Operation: operation, + Duration: duration, + Success: success, + } + if err != nil { + entry.Error = err.Error() + } + r.EnsureTimings = append(r.EnsureTimings, entry) +} + +// AddExecuteTiming adds a timing entry for an execute operation +func (r *TestReport) AddExecuteTiming(operation string, duration time.Duration, success bool, err error) { + entry := TimingEntry{ + Operation: operation, + Duration: duration, + Success: success, + } + if err != nil { + entry.Error = err.Error() + } + r.ExecuteTimings = append(r.ExecuteTimings, entry) +} + +// AddContainerErrors adds error logs for a container +func (r *TestReport) AddContainerErrors(containerName string, errorLines []string) { + if len(errorLines) > 0 { + r.ContainerErrors = append(r.ContainerErrors, ContainerErrorLog{ + ContainerName: containerName, + ErrorLines: errorLines, + }) + } +} + +// SetResult sets the final test result +func (r *TestReport) SetResult(success bool, err error) { + r.EndTime = time.Now() + r.Success = success + if err != nil { + r.ErrorMessage = err.Error() + } +} + +// TotalDuration returns the total test duration +func (r *TestReport) TotalDuration() time.Duration { + return r.EndTime.Sub(r.StartTime) +} + +// ToMarkdown generates a markdown report for GitHub PR comment +func (r *TestReport) ToMarkdown() string { + var sb strings.Builder + + // Header with result emoji + if r.Success { + sb.WriteString("## ✅ Staker Test Report - PASSED\n\n") + } else { + sb.WriteString("## ❌ Staker Test Report - FAILED\n\n") + } + + // Clients Used section + sb.WriteString("### 📦 Clients Used\n\n") + sb.WriteString("| Component | DNP Name |\n") + sb.WriteString("|-----------|----------|\n") + sb.WriteString(fmt.Sprintf("| Execution | `%s` |\n", r.ExecutionDnpName)) + sb.WriteString(fmt.Sprintf("| Consensus | `%s` |\n", r.ConsensusDnpName)) + sb.WriteString(fmt.Sprintf("| Web3Signer | `%s` |\n", r.Web3SignerDnpName)) + sb.WriteString(fmt.Sprintf("| MEV Boost | `%s` |\n", r.MevBoostDnpName)) + sb.WriteString(fmt.Sprintf("| Network | `%s` |\n", r.Network)) + sb.WriteString("\n") + + // Version tracking section + if r.TestedClientType != "" { + sb.WriteString("### 🔖 Version Tracking\n\n") + if r.TestedClientType == "execution" { + sb.WriteString("**Tested Client:** Execution\n\n") + sb.WriteString("| Stage | Version |\n") + sb.WriteString("|-------|---------|\n") + if r.ExecutionClientVersionBefore != "" { + sb.WriteString(fmt.Sprintf("| Before Install | `%s` |\n", r.ExecutionClientVersionBefore)) + } else { + sb.WriteString("| Before Install | _not available_ |\n") + } + if r.ExecutionClientVersionAfter != "" { + sb.WriteString(fmt.Sprintf("| After Install | `%s` |\n", r.ExecutionClientVersionAfter)) + } else { + sb.WriteString("| After Install | _not available_ |\n") + } + if r.SnapshotClientVersion != "" { + sb.WriteString(fmt.Sprintf("| Snapshot | `%s` |\n", r.SnapshotClientVersion)) + } + if r.SnapshotBlockNumber > 0 { + sb.WriteString(fmt.Sprintf("| Snapshot Block | `%d` |\n", r.SnapshotBlockNumber)) + } + } else if r.TestedClientType == "consensus" { + sb.WriteString("**Tested Client:** Consensus\n\n") + sb.WriteString("| Stage | Version |\n") + sb.WriteString("|-------|---------|\n") + if r.ConsensusClientVersionBefore != "" { + sb.WriteString(fmt.Sprintf("| Before Install | `%s` |\n", r.ConsensusClientVersionBefore)) + } else { + sb.WriteString("| Before Install | _not available_ |\n") + } + if r.ConsensusClientVersionAfter != "" { + sb.WriteString(fmt.Sprintf("| After Install | `%s` |\n", r.ConsensusClientVersionAfter)) + } else { + sb.WriteString("| After Install | _not available_ |\n") + } + } + sb.WriteString("\n") + } + + // Timing section + sb.WriteString("### ⏱️ Timing Measurements\n\n") + + if len(r.EnsureTimings) > 0 { + sb.WriteString("#### Environment Setup\n\n") + sb.WriteString("| Operation | Duration | Status |\n") + sb.WriteString("|-----------|----------|--------|\n") + for _, t := range r.EnsureTimings { + status := "✅" + if !t.Success { + status = "❌" + } + sb.WriteString(fmt.Sprintf("| %s | %s | %s |\n", t.Operation, t.Duration.Round(time.Millisecond), status)) + } + sb.WriteString("\n") + } + + if len(r.ExecuteTimings) > 0 { + sb.WriteString("#### Test Execution\n\n") + sb.WriteString("| Operation | Duration | Status |\n") + sb.WriteString("|-----------|----------|--------|\n") + for _, t := range r.ExecuteTimings { + status := "✅" + if !t.Success { + status = "❌" + } + sb.WriteString(fmt.Sprintf("| %s | %s | %s |\n", t.Operation, t.Duration.Round(time.Millisecond), status)) + } + sb.WriteString("\n") + } + + sb.WriteString(fmt.Sprintf("**Total Duration:** %s\n\n", r.TotalDuration().Round(time.Second))) + + // Error logs section + if len(r.ContainerErrors) > 0 { + sb.WriteString("### 🔴 Container Error Logs\n\n") + sb.WriteString("> ⚠️ Showing up to 3 error lines per container. See CI logs for complete details.\n\n") + for _, ce := range r.ContainerErrors { + sb.WriteString(fmt.Sprintf("**%s:**\n", ce.ContainerName)) + sb.WriteString("```\n") + for _, line := range ce.ErrorLines { + sb.WriteString(line + "\n") + } + sb.WriteString("```\n\n") + } + } + + // Error message if failed + if !r.Success && r.ErrorMessage != "" { + sb.WriteString("### ❌ Error Details\n\n") + sb.WriteString("```\n") + sb.WriteString(r.ErrorMessage + "\n") + sb.WriteString("```\n") + } + + return sb.String() +} + +// ToConsoleString generates a console-friendly report +func (r *TestReport) ToConsoleString() string { + var sb strings.Builder + + sb.WriteString("\n") + sb.WriteString("========================================\n") + if r.Success { + sb.WriteString(" STAKER TEST REPORT - PASSED \n") + } else { + sb.WriteString(" STAKER TEST REPORT - FAILED \n") + } + sb.WriteString("========================================\n\n") + + // Clients Used + sb.WriteString("CLIENTS USED:\n") + sb.WriteString(fmt.Sprintf(" Execution: %s\n", r.ExecutionDnpName)) + sb.WriteString(fmt.Sprintf(" Consensus: %s\n", r.ConsensusDnpName)) + sb.WriteString(fmt.Sprintf(" Web3Signer: %s\n", r.Web3SignerDnpName)) + sb.WriteString(fmt.Sprintf(" MEV Boost: %s\n", r.MevBoostDnpName)) + sb.WriteString(fmt.Sprintf(" Network: %s\n", r.Network)) + sb.WriteString("\n") + + // Version tracking + if r.TestedClientType != "" { + sb.WriteString("VERSION TRACKING:\n") + sb.WriteString(fmt.Sprintf(" Tested Client Type: %s\n", r.TestedClientType)) + if r.TestedClientType == "execution" { + if r.ExecutionClientVersionBefore != "" { + sb.WriteString(fmt.Sprintf(" Before Install: %s\n", r.ExecutionClientVersionBefore)) + } + if r.ExecutionClientVersionAfter != "" { + sb.WriteString(fmt.Sprintf(" After Install: %s\n", r.ExecutionClientVersionAfter)) + } + if r.SnapshotClientVersion != "" { + sb.WriteString(fmt.Sprintf(" Snapshot Version: %s\n", r.SnapshotClientVersion)) + } + if r.SnapshotBlockNumber > 0 { + sb.WriteString(fmt.Sprintf(" Snapshot Block: %d\n", r.SnapshotBlockNumber)) + } + } else if r.TestedClientType == "consensus" { + if r.ConsensusClientVersionBefore != "" { + sb.WriteString(fmt.Sprintf(" Before Install: %s\n", r.ConsensusClientVersionBefore)) + } + if r.ConsensusClientVersionAfter != "" { + sb.WriteString(fmt.Sprintf(" After Install: %s\n", r.ConsensusClientVersionAfter)) + } + } + sb.WriteString("\n") + } + + // Timing + sb.WriteString("TIMING MEASUREMENTS:\n") + if len(r.EnsureTimings) > 0 { + sb.WriteString(" Environment Setup:\n") + for _, t := range r.EnsureTimings { + status := "OK" + if !t.Success { + status = "FAILED" + } + sb.WriteString(fmt.Sprintf(" - %-30s %12s [%s]\n", t.Operation, t.Duration.Round(time.Millisecond), status)) + } + } + if len(r.ExecuteTimings) > 0 { + sb.WriteString(" Test Execution:\n") + for _, t := range r.ExecuteTimings { + status := "OK" + if !t.Success { + status = "FAILED" + } + sb.WriteString(fmt.Sprintf(" - %-30s %12s [%s]\n", t.Operation, t.Duration.Round(time.Millisecond), status)) + } + } + sb.WriteString(fmt.Sprintf("\n Total Duration: %s\n", r.TotalDuration().Round(time.Second))) + sb.WriteString("\n") + + // Error logs + if len(r.ContainerErrors) > 0 { + sb.WriteString("CONTAINER ERROR LOGS:\n") + sb.WriteString(" (See CI logs for complete details)\n") + for _, ce := range r.ContainerErrors { + sb.WriteString(fmt.Sprintf(" [%s]:\n", ce.ContainerName)) + for _, line := range ce.ErrorLines { + sb.WriteString(fmt.Sprintf(" %s\n", line)) + } + } + sb.WriteString("\n") + } + + // Error message + if !r.Success && r.ErrorMessage != "" { + sb.WriteString("ERROR DETAILS:\n") + sb.WriteString(fmt.Sprintf(" %s\n", r.ErrorMessage)) + } + + sb.WriteString("========================================\n") + + return sb.String() +} diff --git a/internal/application/domain/snapshot.go b/internal/application/domain/snapshot.go new file mode 100644 index 0000000..d7b2b51 --- /dev/null +++ b/internal/application/domain/snapshot.go @@ -0,0 +1,88 @@ +package domain + +import ( + "fmt" +) + +// ExecutionClientInfo contains the information for an execution client +type ExecutionClientInfo struct { + ShortName string // geth, nethermind, reth, besu, erigon + DnpName string // hoodi-nethermind.dnp.dappnode.eth + VolumeName string // Docker volume name + ContainerName string // Docker container name + VolumeTargetPath string // Path to the volume data (/var/lib/docker/volumes/...) +} + +// SnapshotCheckerConfig contains the configuration for the snapshot checker +type SnapshotCheckerConfig struct { + ExecutionClient ExecutionClientInfo // The execution client to manage + CronIntervalSec int // Interval between snapshot checks in seconds (default 6 hours) + Network string // Network name (e.g., hoodi) +} + +// ValidExecutionClients contains all valid execution client short names for hoodi +var ValidExecutionClients = []string{"geth", "nethermind", "reth", "besu", "erigon"} + +// SnapshotProgressPath is the directory for progress files +const SnapshotProgressPath = "/usr/src/dappnode/DNCORE" + +// ProgressFileName is the name of the download in progress file +const ProgressFileName = ".download_in_progress" + +// TestProgressFileName is the name of the test in progress file +const TestProgressFileName = ".test_in_progress" + +// SnapshotBlockNumberFileName returns the snapshot block number file name +const SnapshotBlockNumberFileName = "snapshot_block_number" + +// GetExecutionClient returns the execution client info for a specific client on hoodi network +func GetExecutionClient(network string, clientName string) (ExecutionClientInfo, bool) { + allClients := map[string]string{ + "reth": "hoodi-reth.dnp.dappnode.eth", + "geth": "hoodi-geth.dnp.dappnode.eth", + // "besu": "hoodi-besu.dnp.dappnode.eth", + // "erigon": "hoodi-erigon.dnp.dappnode.eth", + "nethermind": "hoodi-nethermind.dnp.dappnode.eth", + } + + dnpName, exists := allClients[clientName] + if !exists { + return ExecutionClientInfo{}, false + } + + serviceName := serviceNameFromExecutionClient(dnpName, network) + // reth and geth use different volume names than data + // - reth: https://github.com/dappnode/DAppNodePackage-reth-generic/blob/0de584dafd9b07fe24090f7e1cb96aa7f9108769/docker-compose.yml#L18 + // - geth: https://github.com/dappnode/DAppNodePackage-geth-generic/blob/1b9ed0da445e8599e7182a083cb3ffc98bf6b289/package_variants/hoodi/docker-compose.yml#L16 + var volumeArg string + if clientName == "geth" || clientName == "reth" { + volumeArg = clientName + } else { + volumeArg = "data" + } + volumeName := composeVolumeName(dnpName, volumeArg) + volumeTargetPath := fmt.Sprintf("/var/lib/docker/volumes/%s/_data", volumeName) + + return ExecutionClientInfo{ + ShortName: clientName, + DnpName: dnpName, + VolumeName: volumeName, + ContainerName: containerName(serviceName, dnpName), + VolumeTargetPath: volumeTargetPath, + }, true +} + +// IsValidExecutionClient checks if a client name is valid +func IsValidExecutionClient(client string) bool { + return containsString(ValidExecutionClients, client) +} + +// containsString checks if a string slice contains a value +func containsString(slice []string, value string) bool { + for _, v := range slice { + if v == value { + return true + } + } + return false +} diff --git a/internal/application/domain/stakers.go b/internal/application/domain/stakers.go index 29b4206..531c495 100644 --- a/internal/application/domain/stakers.go +++ b/internal/application/domain/stakers.go @@ -8,16 +8,19 @@ import ( ) type StakerConfig struct { - ExecutionDnpName string `json:"executionDnpName"` - ConsensusDnpName string `json:"consensusDnpName"` - Web3SignerDnpName string `json:"web3signerDnpName"` - MevBoostDnpName string `json:"mevBoostDnpName"` - Relays []string `json:"relays,omitempty"` // Optional, can be empty - Network string `json:"network"` // The network this config is for (e.g., mainnet, gnosis, hoodi, lukso) - Urls Urls - ExecutionContainerName string // The name of the container to mount the NFS volume to - ExecutionVolumeName string // The name of the volume to mount for execution client data - DataBackendName string // The name of the backend to use for data requests + ExecutionDnpName string `json:"executionDnpName"` + ConsensusDnpName string `json:"consensusDnpName"` + Web3SignerDnpName string `json:"web3signerDnpName"` + MevBoostDnpName string `json:"mevBoostDnpName"` + Relays []string `json:"relays,omitempty"` // Optional, can be empty + Network string `json:"network"` // The network this config is for (e.g., mainnet, gnosis, hoodi, lukso) + Urls Urls + BrainContainerName string // The name of the brain container + SignerContainerName string // The name of the web3signer container + BeaconchainContainerName string // The name of the beaconchain container + ValidatorContainerName string // The name of the validator container + ExecutionContainerName string // The name of the container to mount the NFS volume to + ExecutionVolumeTargetPath string // Path to the execution client's docker volume data } type Urls struct { @@ -27,112 +30,152 @@ type Urls struct { DappmanagerURL string } -func StakerConfigForNetwork(pkg Pkg) StakerConfig { - network := getNetworkFromDnpName(pkg.DnpName) - var execClients, consClients []string - var web3signer, mevboost string - var relays []string = nil - var urls Urls - - switch network { - case "gnosis": - execClients = []string{"nethermind-xdai.dnp.dappnode.eth", "gnosis-erigon.dnp.dappnode.eth"} - consClients = []string{"lighthouse-gnosis.dnp.dappnode.eth", "teku-gnosis.dnp.dappnode.eth", "nimbus-gnosis.dnp.dappnode.eth", "lodestar-gnosis.dnp.dappnode.eth"} - web3signer = "web3signer-hoodi.dnp.dappnode.eth" - mevboost = "mev-boost-hoodi.dnp.dappnode.eth" - relays = []string{} - urls = Urls{ - ExecutionURL: "http://execution.gnosis.dncore.dappnode:8545", - BrainURL: "http://brain.web3signer-gnosis.dappnode:5000", - BeaconchainURL: "http://beacon-chain.gnosis.dncore.dappnode:3500", - DappmanagerURL: "http://dappmanager.dappnode:7000", - } - case "mainnet": - execClients = []string{"nethermind.public.dappnode.eth", "geth.dnp.dappnode.eth", "erigon.dnp.dappnode.eth", "reth.dnp.dappnode.eth", "besu.public.dappnode.eth"} - consClients = []string{"lighthouse.dnp.dappnode.eth", "prysm.dnp.dappnode.eth", "lodestar.dnp.dappnode.eth", "nimbus.dnp.dappnode.eth", "teku.dnp.dappnode.eth"} - web3signer = "web3signer.dnp.dappnode.eth" - mevboost = "mev-boost.dnp.dappnode.eth" - relays = []string{} - urls = Urls{ - ExecutionURL: "http://execution.mainnet.dncore.dappnode:8545", - BrainURL: "http://brain.web3signer.dappnode:5000", - BeaconchainURL: "http://beacon-chain.mainnet.dncore.dappnode:3500", - DappmanagerURL: "http://dappmanager.dappnode:7000", - } - case "lukso": - execClients = []string{"lukso-geth.dnp.dappnode.eth"} - consClients = []string{"prysm-lukso.dnp.dappnode.eth", "teku-luks.dnp.dappnode.eth"} - web3signer = "web3signer-lukso.dnp.dappnode.eth" - mevboost = "mev-boost-lukso.dnp.dappnode.eth" - relays = []string{} - urls = Urls{ - ExecutionURL: "http://execution.lukso.dncore.dappnode:8545", - BrainURL: "http://brain.web3signer-lukso.dappnode:5000", - BeaconchainURL: "http://beacon-chain.lukso.dncore.dappnode:3500", - DappmanagerURL: "http://dappmanager.dappnode:7000", - } - case "hoodi": - execClients = []string{"hoodi-reth.dnp.dappnode.eth", "hoodi-geth.dnp.dappnode.eth", "hoodi-besu.dnp.dappnode.eth", "hoodi-erigon.dnp.dappnode.eth", "hoodi-nethermind.dnp.dappnode.eth"} - consClients = []string{"prysm-hoodi.dnp.dappnode.eth", "teku-hoodi.dnp.dappnode.eth", "nimbus-hoodi.dnp.dappnode.eth", "lodestar-hoodi.dnp.dappnode.eth", "lighthouse-hoodi.dnp.dappnode.eth"} - web3signer = "web3signer-hoodi.dnp.dappnode.eth" - mevboost = "mev-boost-hoodi.dnp.dappnode.eth" - relays = []string{} - urls = Urls{ - ExecutionURL: "http://execution.hoodi.dncore.dappnode:8545", - BrainURL: "http://brain.web3signer-hoodi.dappnode:5000", - BeaconchainURL: "http://beacon-chain.hoodi.dncore.dappnode:3500", - DappmanagerURL: "http://dappmanager.dappnode:8080", - } +// ClientOverrides holds optional client override settings +type ClientOverrides struct { + ExecutionClient string // Short name like "geth", "reth", etc. + ConsensusClient string // Short name like "prysm", "teku", etc. +} + +// ClientOverrideResult holds the result of applying overrides with any warnings +type ClientOverrideResult struct { + ExecutionDnpName string + ConsensusDnpName string + Warnings []string +} + +const dappmanagerURL = "http://dappmanager.dappnode:7000" + +// hoodi network client lists +var ( + hoodiExecClients = []string{"hoodi-reth.dnp.dappnode.eth", "hoodi-geth.dnp.dappnode.eth", "hoodi-besu.dnp.dappnode.eth", "hoodi-erigon.dnp.dappnode.eth", "hoodi-nethermind.dnp.dappnode.eth"} + hoodiConsClients = []string{"prysm-hoodi.dnp.dappnode.eth", "teku-hoodi.dnp.dappnode.eth", "nimbus-hoodi.dnp.dappnode.eth", "lodestar-hoodi.dnp.dappnode.eth"} +) + +func StakerConfigForNetwork(pkg Pkg, overrides ClientOverrides) (StakerConfig, []string) { + // Only hoodi network is supported + network := "hoodi" + web3signer := "web3signer-hoodi.dnp.dappnode.eth" + mevboost := "mev-boost-hoodi.dnp.dappnode.eth" + relays := []string{} + urls := Urls{ + ExecutionURL: "http://execution.hoodi.dncore.dappnode:8545", + BrainURL: "http://brain.web3signer-hoodi.dappnode:5000", + BeaconchainURL: "http://beacon-chain.hoodi.dncore.dappnode:3500", + DappmanagerURL: dappmanagerURL, } - exec := matchOrRandom(pkg.DnpName, execClients) - cons := matchOrRandom(pkg.DnpName, consClients) + // Resolve execution and consensus clients with override logic + result := resolveClientsWithOverrides(pkg, overrides, hoodiExecClients, hoodiConsClients) + + ecDnpName := result.ExecutionDnpName + ccDnpName := result.ConsensusDnpName + + serviceName := serviceNameFromExecutionClient(ecDnpName, network) + volumeName := getExecutionVolumeName(ecDnpName, serviceName) + + return StakerConfig{ + ExecutionDnpName: ecDnpName, + ConsensusDnpName: ccDnpName, + Web3SignerDnpName: web3signer, + MevBoostDnpName: mevboost, + Relays: relays, + Network: network, + Urls: urls, + BrainContainerName: containerName("brain", web3signer), + SignerContainerName: containerName("web3signer", web3signer), + BeaconchainContainerName: containerName("beacon-chain", ccDnpName), + ValidatorContainerName: containerName("validator", ccDnpName), + ExecutionContainerName: containerName(serviceName, ecDnpName), + ExecutionVolumeTargetPath: fmt.Sprintf("/var/lib/docker/volumes/%s/_data", volumeName), + }, result.Warnings +} + +// resolveClientsWithOverrides determines execution and consensus clients based on: +// 1. If pkg matches an execution/consensus client, use it (overriding any flag/env with warning) +// 2. Otherwise, use the override if provided +// 3. Otherwise, pick a random client +func resolveClientsWithOverrides(pkg Pkg, overrides ClientOverrides, execClients, consClients []string) ClientOverrideResult { + result := ClientOverrideResult{} - // List of known execution client short names - clientShortNames := []string{"geth", "nethermind", "erigon", "reth", "besu"} - execShort := "unknown" - for _, short := range clientShortNames { - if strings.Contains(exec, short) { - execShort = short - break + // Check if pkg is an execution client + pkgMatchedExec := matchClient(pkg.DnpName, execClients) + // Check if pkg is a consensus client + pkgMatchedCons := matchClient(pkg.DnpName, consClients) + + // Resolve execution client + if pkgMatchedExec != "" { + // Pkg is an execution client - use it + if overrides.ExecutionClient != "" { + result.Warnings = append(result.Warnings, + fmt.Sprintf("Package '%s' is an execution client; ignoring --execution-client flag '%s'", + pkg.DnpName, overrides.ExecutionClient)) + } + result.ExecutionDnpName = pkgMatchedExec + } else if overrides.ExecutionClient != "" { + // Use override if provided + matched := matchClientByShortName(overrides.ExecutionClient, execClients) + if matched != "" { + result.ExecutionDnpName = matched + } else { + result.Warnings = append(result.Warnings, + fmt.Sprintf("Unknown execution client '%s'; using random", overrides.ExecutionClient)) + result.ExecutionDnpName = randomClient(execClients) } + } else { + // Pick random + result.ExecutionDnpName = randomClient(execClients) } - dataBackend := execShort + "-" + network - return StakerConfig{ - ExecutionDnpName: exec, - ConsensusDnpName: cons, - Web3SignerDnpName: web3signer, - MevBoostDnpName: mevboost, - Relays: relays, - Network: network, - Urls: urls, - ExecutionContainerName: executionContainerName(pkg.ServiceName, pkg.DnpName), - ExecutionVolumeName: composeVolumeName(pkg.DnpName, pkg.ComposeVolumeName), - DataBackendName: dataBackend, + // Resolve consensus client + if pkgMatchedCons != "" { + // Pkg is a consensus client - use it + if overrides.ConsensusClient != "" { + result.Warnings = append(result.Warnings, + fmt.Sprintf("Package '%s' is a consensus client; ignoring --consensus-client flag '%s'", + pkg.DnpName, overrides.ConsensusClient)) + } + result.ConsensusDnpName = pkgMatchedCons + } else if overrides.ConsensusClient != "" { + // Use override if provided + matched := matchClientByShortName(overrides.ConsensusClient, consClients) + if matched != "" { + result.ConsensusDnpName = matched + } else { + result.Warnings = append(result.Warnings, + fmt.Sprintf("Unknown consensus client '%s'; using random", overrides.ConsensusClient)) + result.ConsensusDnpName = randomClient(consClients) + } + } else { + // Pick random + result.ConsensusDnpName = randomClient(consClients) } + + return result } -func getNetworkFromDnpName(dnpName string) string { - name := strings.ToLower(dnpName) - switch { - case strings.Contains(name, "gnosis"): - return "gnosis" - case strings.Contains(name, "hoodi"): - return "hoodi" - case strings.Contains(name, "lukso"): - return "lukso" - default: - return "mainnet" +// matchClient returns the matching client dnpName if found, empty string otherwise +func matchClient(dnpName string, candidates []string) string { + for _, c := range candidates { + if strings.Contains(dnpName, c) || strings.Contains(c, dnpName) { + return c + } } + return "" } -func matchOrRandom(dnpName string, candidates []string) string { +// matchClientByShortName matches a short name (e.g., "geth", "prysm") to a full dnpName +func matchClientByShortName(shortName string, candidates []string) string { + shortName = strings.ToLower(shortName) for _, c := range candidates { - if strings.Contains(dnpName, c) { + if strings.Contains(strings.ToLower(c), shortName) { return c } } + return "" +} + +// randomClient picks a random client from the list +func randomClient(candidates []string) string { if len(candidates) == 0 { return "" } @@ -140,18 +183,14 @@ func matchOrRandom(dnpName string, candidates []string) string { return candidates[r.Intn(len(candidates))] } -// Utility to get the short dnp name (strip .dnp.dappnode.eth) -func shortDnpName(dnpName string) string { - return strings.TrimSuffix(dnpName, ".dnp.dappnode.eth") -} - -// Utility to get the execution container name from service and dnpName -func executionContainerName(serviceName, dnpName string) string { - return fmt.Sprintf("DAppNodePackage-%s.%s.dnp.dappnode.eth", serviceName, shortDnpName(dnpName)) -} - -// Utility to get the docker volume name from dnpName and compose volume name -// i.e hoodi-nethermind.dnp.dappnode.eth -> hoodi-netherminddnpdappnodeeth_ -func composeVolumeName(dnpName, composeVolumeName string) string { - return fmt.Sprintf("%s_%s", strings.ReplaceAll(dnpName, ".", ""), composeVolumeName) +// getExecutionVolumeName returns the docker volume name for the execution client +// reth and geth use their service name as volume name, others use "data" +func getExecutionVolumeName(dnpName, serviceName string) string { + var volumeArg string + if serviceName == "geth" || serviceName == "reth" { + volumeArg = serviceName + } else { + volumeArg = "data" + } + return composeVolumeName(dnpName, volumeArg) } diff --git a/internal/application/ports/blockNumber_port.go b/internal/application/ports/blockNumber_port.go new file mode 100644 index 0000000..979ca8b --- /dev/null +++ b/internal/application/ports/blockNumber_port.go @@ -0,0 +1,20 @@ +package ports + +import "context" + +// BlockNumber defines the interface for block number tracking operations +// Used to track the current snapshot block number and compare with available snapshots +type BlockNumber interface { + // ReadBlockNumber reads the current block number from storage + // Returns empty string if no block number file exists + ReadBlockNumber(ctx context.Context) (string, error) + + // WriteBlockNumber writes the block number to storage + WriteBlockNumber(ctx context.Context, blockNumber string) error + + // BlockNumberExists checks if a block number file exists + BlockNumberExists(ctx context.Context) (bool, error) + + // IsNewerSnapshot checks if the latest available block number is newer than the current one + IsNewerSnapshot(ctx context.Context, latestBlockNumber string) (bool, error) +} diff --git a/internal/application/ports/downloadProgress_port.go b/internal/application/ports/downloadProgress_port.go new file mode 100644 index 0000000..f25f4ff --- /dev/null +++ b/internal/application/ports/downloadProgress_port.go @@ -0,0 +1,16 @@ +package ports + +import "context" + +// DownloadProgress defines the interface for tracking download progress state +// Used to prevent concurrent downloads and recover from interrupted downloads +type DownloadProgress interface { + // IsDownloadInProgress checks if a download is currently in progress + IsDownloadInProgress(ctx context.Context) (bool, error) + + // SetDownloadInProgress marks that a download has started + SetDownloadInProgress(ctx context.Context) error + + // ClearDownloadInProgress marks that a download has completed (success or failure) + ClearDownloadInProgress(ctx context.Context) error +} diff --git a/internal/application/ports/ports.go b/internal/application/ports/ports.go deleted file mode 100644 index 23a5d2b..0000000 --- a/internal/application/ports/ports.go +++ /dev/null @@ -1,12 +0,0 @@ -package ports - -import ( - "clients-test/internal/application/domain" - "context" -) - -type TestRunner interface { - EnsureEnvironment(ctx context.Context, mountConfig domain.Mount, stakerConfig domain.StakerConfig, pkg domain.Pkg) error - ExecuteTest(ctx context.Context) error - CleanEnvironment(context.Context, domain.StakerConfig, domain.Mount) error -} diff --git a/internal/application/ports/snapshotChecker_port.go b/internal/application/ports/snapshotChecker_port.go new file mode 100644 index 0000000..3c1773b --- /dev/null +++ b/internal/application/ports/snapshotChecker_port.go @@ -0,0 +1,19 @@ +package ports + +import ( + "clients-test/internal/application/domain" + "context" +) + +// SnapshotManager defines the interface for snapshot management operations +// This is the port that the snapshot checker service uses to interact with snapshot infrastructure +type SnapshotManager interface { + // DownloadAndMountSnapshot performs the complete snapshot download and mount process + DownloadAndMountSnapshot(ctx context.Context, network string, client domain.ExecutionClientInfo) error + + // GetLatestBlockNumber fetches the latest available block number for a client + GetLatestBlockNumber(ctx context.Context, network, client string) (string, error) + + // StopDownload stops the snapshot download container for a specific client + StopDownload(ctx context.Context, clientShortName string) +} diff --git a/internal/application/ports/testProgress_port.go b/internal/application/ports/testProgress_port.go new file mode 100644 index 0000000..a2708c6 --- /dev/null +++ b/internal/application/ports/testProgress_port.go @@ -0,0 +1,16 @@ +package ports + +import "context" + +// TestProgress defines the interface for tracking test progress state +// Used to prevent snapshot downloads while a test is running +type TestProgress interface { + // IsTestInProgress checks if a test is currently in progress + IsTestInProgress(ctx context.Context) (bool, error) + + // SetTestInProgress marks that a test has started + SetTestInProgress(ctx context.Context) error + + // ClearTestInProgress marks that a test has completed (success or failure) + ClearTestInProgress(ctx context.Context) error +} diff --git a/internal/application/ports/testRunner_port.go b/internal/application/ports/testRunner_port.go new file mode 100644 index 0000000..0b932af --- /dev/null +++ b/internal/application/ports/testRunner_port.go @@ -0,0 +1,12 @@ +package ports + +import ( + "clients-test/internal/application/domain" + "context" +) + +type TestRunner interface { + EnsureEnvironment(ctx context.Context, stakerConfig domain.StakerConfig, pkg domain.Pkg) error + ExecuteTest(ctx context.Context, stakerConfig domain.StakerConfig) error + CleanEnvironment(context.Context, domain.StakerConfig) error +} diff --git a/internal/application/services/snapshot_checker.go b/internal/application/services/snapshot_checker.go new file mode 100644 index 0000000..3ae2616 --- /dev/null +++ b/internal/application/services/snapshot_checker.go @@ -0,0 +1,246 @@ +package services + +import ( + "clients-test/internal/application/domain" + "clients-test/internal/application/ports" + "clients-test/internal/logger" + "context" + "fmt" + "time" +) + +var snapshotLogPrefix = "SnapshotChecker" + +// SnapshotCheckerService handles snapshot checking and downloading +type SnapshotCheckerService struct { + snapshotManager ports.SnapshotManager + downloadProgress ports.DownloadProgress + testProgress ports.TestProgress + blockNumber ports.BlockNumber + config domain.SnapshotCheckerConfig +} + +// NewSnapshotCheckerService creates a new SnapshotCheckerService +func NewSnapshotCheckerService( + snapshotManager ports.SnapshotManager, + downloadProgress ports.DownloadProgress, + testProgress ports.TestProgress, + blockNumber ports.BlockNumber, + config domain.SnapshotCheckerConfig, +) *SnapshotCheckerService { + return &SnapshotCheckerService{ + snapshotManager: snapshotManager, + downloadProgress: downloadProgress, + testProgress: testProgress, + blockNumber: blockNumber, + config: config, + } +} + +// GetSnapshotManager returns the snapshot manager adapter (for shutdown handling) +func (s *SnapshotCheckerService) GetSnapshotManager() ports.SnapshotManager { + return s.snapshotManager +} + +// Start starts the snapshot checker cron job +func (s *SnapshotCheckerService) Start(ctx context.Context, runOnce bool) error { + logger.InfoWithPrefix(snapshotLogPrefix, "Starting snapshot checker for network: %s", s.config.Network) + logger.InfoWithPrefix(snapshotLogPrefix, "Managing execution client: %s", s.config.ExecutionClient.ShortName) + + // Run immediately on startup + logger.InfoWithPrefix(snapshotLogPrefix, "Running initial snapshot check...") + if err := s.checkAndUpdateSnapshot(ctx); err != nil { + logger.ErrorWithPrefix(snapshotLogPrefix, "Initial snapshot check failed: %v", err) + return err + } + + if runOnce { + logger.InfoWithPrefix(snapshotLogPrefix, "Run-once mode enabled, exiting after initial check") + return nil + } + + // Start cron loop + ticker := time.NewTicker(time.Duration(s.config.CronIntervalSec) * time.Second) + defer ticker.Stop() + + logger.InfoWithPrefix(snapshotLogPrefix, "Cron job scheduled every %d seconds (%s)", + s.config.CronIntervalSec, time.Duration(s.config.CronIntervalSec)*time.Second) + + for { + select { + case <-ctx.Done(): + logger.InfoWithPrefix(snapshotLogPrefix, "Snapshot checker stopped: %v", ctx.Err()) + return ctx.Err() + case <-ticker.C: + logger.InfoWithPrefix(snapshotLogPrefix, "Running scheduled snapshot check...") + if err := s.checkAndUpdateSnapshot(ctx); err != nil { + logger.ErrorWithPrefix(snapshotLogPrefix, "Scheduled snapshot check failed: %v", err) + // Continue running even on failure + } + } + } +} + +// checkAndUpdateSnapshot checks the configured client and updates snapshot if needed +func (s *SnapshotCheckerService) checkAndUpdateSnapshot(ctx context.Context) error { + client := s.config.ExecutionClient + logger.InfoWithPrefix(snapshotLogPrefix, "[%s] Checking client (%s)", client.ShortName, client.DnpName) + + // Wait for any ongoing tests to complete before proceeding with this client + logger.InfoWithPrefix(snapshotLogPrefix, "[%s] Checking if test is in progress...", client.ShortName) + if err := s.waitForTestCompleteWithRetry(ctx, client.ShortName); err != nil { + return fmt.Errorf("error while waiting for test to complete: %w", err) + } + logger.InfoWithPrefix(snapshotLogPrefix, "[%s] No ongoing tests detected", client.ShortName) + + // Check if a download is already in progress for this client + logger.InfoWithPrefix(snapshotLogPrefix, "[%s] Checking if download is already in progress...", client.ShortName) + inProgress, err := s.downloadProgress.IsDownloadInProgress(ctx) + if err != nil { + return fmt.Errorf("failed to check download progress: %w", err) + } + if inProgress { + logger.InfoWithPrefix(snapshotLogPrefix, "[%s] Download already in progress, skipping", client.ShortName) + return nil + } + + // Get latest available block number from ethpandaops + logger.InfoWithPrefix(snapshotLogPrefix, "[%s] Fetching latest available block number...", client.ShortName) + latestBlockNumber, err := s.snapshotManager.GetLatestBlockNumber(ctx, s.config.Network, client.ShortName) + if err != nil { + return fmt.Errorf("failed to get latest block number: %w", err) + } + logger.InfoWithPrefix(snapshotLogPrefix, "[%s] Latest available snapshot block: %s", client.ShortName, latestBlockNumber) + + // Check if we need to download by reading current block number + logger.InfoWithPrefix(snapshotLogPrefix, "[%s] Checking current snapshot block number...", client.ShortName) + needsDownload, err := s.checkNeedsSnapshotDownload(ctx, client, latestBlockNumber) + if err != nil { + return fmt.Errorf("failed to check if snapshot needed: %w", err) + } + + if !needsDownload { + logger.InfoWithPrefix(snapshotLogPrefix, "[%s] Snapshot is up to date, skipping", client.ShortName) + return nil + } + + logger.InfoWithPrefix(snapshotLogPrefix, "[%s] Snapshot download needed", client.ShortName) + + // Set download in progress for this client + logger.InfoWithPrefix(snapshotLogPrefix, "[%s] Setting download in progress marker...", client.ShortName) + if err := s.downloadProgress.SetDownloadInProgress(ctx); err != nil { + return fmt.Errorf("failed to set download in progress: %w", err) + } + + // Ensure we clear the progress file on completion (success or failure) + defer func() { + logger.InfoWithPrefix(snapshotLogPrefix, "[%s] Clearing download in progress marker...", client.ShortName) + if err := s.downloadProgress.ClearDownloadInProgress(ctx); err != nil { + logger.ErrorWithPrefix(snapshotLogPrefix, "[%s] Failed to clear download in progress: %v", client.ShortName, err) + } + }() + + // Download and mount snapshot + logger.InfoWithPrefix(snapshotLogPrefix, "[%s] Starting snapshot download and extraction...", client.ShortName) + logger.InfoWithPrefix(snapshotLogPrefix, "[%s] Target path: %s", client.ShortName, client.VolumeTargetPath) + + start := time.Now() + err = s.snapshotManager.DownloadAndMountSnapshot(ctx, s.config.Network, client) + elapsed := time.Since(start) + + if err != nil { + return fmt.Errorf("failed to download and mount snapshot: %w", err) + } + + // Write block number file after successful download + logger.InfoWithPrefix(snapshotLogPrefix, "[%s] Writing snapshot block number: %s", client.ShortName, latestBlockNumber) + if err := s.blockNumber.WriteBlockNumber(ctx, latestBlockNumber); err != nil { + return fmt.Errorf("failed to write block number: %w", err) + } + + logger.InfoWithPrefix(snapshotLogPrefix, "[%s] ✓ Snapshot download completed successfully", client.ShortName) + logger.InfoWithPrefix(snapshotLogPrefix, "[%s] Total time: %s", client.ShortName, elapsed.Round(time.Second)) + + logger.InfoWithPrefix(snapshotLogPrefix, "Snapshot check completed successfully") + return nil +} + +// checkNeedsSnapshotDownload determines if a snapshot needs to be downloaded +// by reading the current block number and comparing with the latest available +// Logs clearly the current and latest block numbers for visibility +func (s *SnapshotCheckerService) checkNeedsSnapshotDownload(ctx context.Context, client domain.ExecutionClientInfo, latestBlockNumber string) (bool, error) { + // Check if block number file exists + exists, err := s.blockNumber.BlockNumberExists(ctx) + if err != nil { + return false, fmt.Errorf("failed to check if block number exists: %w", err) + } + + if !exists { + logger.InfoWithPrefix(snapshotLogPrefix, "[%s] No snapshot block number file found - download required", client.ShortName) + return true, nil + } + + // Read current block number + currentBlockNumber, err := s.blockNumber.ReadBlockNumber(ctx) + if err != nil { + return false, fmt.Errorf("failed to read current block number: %w", err) + } + + // Log both block numbers clearly + logger.InfoWithPrefix(snapshotLogPrefix, "[%s] Current snapshot block: %s", client.ShortName, currentBlockNumber) + logger.InfoWithPrefix(snapshotLogPrefix, "[%s] Latest available block: %s", client.ShortName, latestBlockNumber) + + // Check if newer snapshot is available + isNewer, err := s.blockNumber.IsNewerSnapshot(ctx, latestBlockNumber) + if err != nil { + return false, fmt.Errorf("failed to compare block numbers: %w", err) + } + + if isNewer { + logger.InfoWithPrefix(snapshotLogPrefix, "[%s] Newer snapshot available (%s > %s) - download required", client.ShortName, latestBlockNumber, currentBlockNumber) + } else { + logger.InfoWithPrefix(snapshotLogPrefix, "[%s] Current snapshot is up to date (current: %s, latest: %s)", client.ShortName, currentBlockNumber, latestBlockNumber) + } + + return isNewer, nil +} + +// StopDownload stops the current download container (for graceful shutdown) +func (s *SnapshotCheckerService) StopDownload(ctx context.Context) { + clientName := s.config.ExecutionClient.ShortName + logger.InfoWithPrefix(snapshotLogPrefix, "[%s] Stopping snapshot download container...", clientName) + s.snapshotManager.StopDownload(ctx, clientName) + logger.InfoWithPrefix(snapshotLogPrefix, "[%s] Download container stopped", clientName) +} + +// ClearDownloadMarker clears the download in progress marker +func (s *SnapshotCheckerService) ClearDownloadMarker(ctx context.Context) { + if err := s.downloadProgress.ClearDownloadInProgress(ctx); err != nil { + logger.WarnWithPrefix(snapshotLogPrefix, "Failed to clear download marker: %v", err) + } +} + +// waitForTestCompleteWithRetry waits until no test is in progress +// using a fixed 30 second retry interval. +func (s *SnapshotCheckerService) waitForTestCompleteWithRetry(ctx context.Context, clientName string) error { + const retryInterval = 30 * time.Second + + for { + select { + case <-ctx.Done(): + return ctx.Err() + default: + inProgress, err := s.testProgress.IsTestInProgress(ctx) + if err != nil { + return fmt.Errorf("error checking test progress: %w", err) + } + + if !inProgress { + return nil + } + + logger.InfoWithPrefix(snapshotLogPrefix, "[%s] Test in progress, waiting %s before retry...", clientName, retryInterval) + time.Sleep(retryInterval) + } + } +} diff --git a/internal/application/services/test_runner.go b/internal/application/services/test_runner.go index 13b1339..2d14b4e 100644 --- a/internal/application/services/test_runner.go +++ b/internal/application/services/test_runner.go @@ -3,40 +3,98 @@ package services import ( "clients-test/internal/application/domain" "clients-test/internal/application/ports" + "clients-test/internal/logger" "context" "fmt" + "time" ) +var logPrefix = "TestRunner" + type TestRunnerService struct { - Runner ports.TestRunner + Runner ports.TestRunner + DownloadProgress ports.DownloadProgress + TestProgress ports.TestProgress } -func NewTestRunner(runner ports.TestRunner) *TestRunnerService { - return &TestRunnerService{runner} +func NewTestRunner(runner ports.TestRunner, downloadProgress ports.DownloadProgress, testProgress ports.TestProgress) *TestRunnerService { + return &TestRunnerService{Runner: runner, DownloadProgress: downloadProgress, TestProgress: testProgress} } // RunTest wires up the three steps in sequence. // It ensures the environment, executes the test, and cleans up. -func (s *TestRunnerService) RunTest(ctx context.Context, mountConfig domain.Mount, stakerConfig domain.StakerConfig, pkg domain.Pkg) error { - // 1) ensure environment - if err := s.Runner.EnsureEnvironment(ctx, mountConfig, stakerConfig, pkg); err != nil { - return fmt.Errorf("setup failed: %w", err) +func (s *TestRunnerService) RunTest(ctx context.Context, stakerConfig domain.StakerConfig, pkg domain.Pkg) error { + // 1) wait for any ongoing downloads to complete + logger.InfoWithPrefix(logPrefix, "Waiting for any ongoing downloads to complete...") + if err := s.WaitForDownloadCompleteWithRetry(ctx); err != nil { + logger.ErrorWithPrefix(logPrefix, "Error while waiting for downloads to complete: %v", err) + return fmt.Errorf("error while waiting for downloads to complete: %w", err) + } + logger.InfoWithPrefix(logPrefix, "No ongoing downloads detected, proceeding with test run") + + // Set test in progress marker + logger.InfoWithPrefix(logPrefix, "Setting test in progress marker...") + if err := s.TestProgress.SetTestInProgress(ctx); err != nil { + logger.ErrorWithPrefix(logPrefix, "Failed to set test in progress: %v", err) + return fmt.Errorf("failed to set test in progress: %w", err) } - // 2) execute test - execErr := s.Runner.ExecuteTest(ctx) - if execErr != nil { - // even on test failure we want cleanup - if cleanupErr := s.Runner.CleanEnvironment(ctx, stakerConfig, mountConfig); cleanupErr != nil { - return fmt.Errorf("test failed: %v; cleanup also failed: %w", execErr, cleanupErr) + // Ensure we clean up and clear the test in progress marker on completion (success or failure) + defer func() { + logger.InfoWithPrefix(logPrefix, "Cleaning up environment...") + if cleanupErr := s.Runner.CleanEnvironment(ctx, stakerConfig); cleanupErr != nil { + logger.ErrorWithPrefix(logPrefix, "Cleanup failed: %v", cleanupErr) + } else { + logger.InfoWithPrefix(logPrefix, "Cleanup completed successfully") } - return fmt.Errorf("test failed: %w", execErr) + + logger.InfoWithPrefix(logPrefix, "Clearing test in progress marker...") + if err := s.TestProgress.ClearTestInProgress(ctx); err != nil { + logger.ErrorWithPrefix(logPrefix, "Failed to clear test in progress: %v", err) + } + }() + + // 2) ensure environment + logger.InfoWithPrefix(logPrefix, "Step 2: Ensuring environment for package %s", pkg.DnpName) + if err := s.Runner.EnsureEnvironment(ctx, stakerConfig, pkg); err != nil { + logger.ErrorWithPrefix(logPrefix, "Setup failed: %v", err) + return fmt.Errorf("setup failed: %w", err) } + logger.InfoWithPrefix(logPrefix, "Environment setup completed successfully") - // 3) cleanup - if err := s.Runner.CleanEnvironment(ctx, stakerConfig, mountConfig); err != nil { - return fmt.Errorf("cleanup failed: %w", err) + // 3) execute test (now includes report generation and PR commenting) + logger.InfoWithPrefix(logPrefix, "Step 3: Executing test") + if err := s.Runner.ExecuteTest(ctx, stakerConfig); err != nil { + logger.ErrorWithPrefix(logPrefix, "Test execution failed: %v", err) + return fmt.Errorf("test failed: %w", err) } + logger.InfoWithPrefix(logPrefix, "Test execution completed successfully") + logger.InfoWithPrefix(logPrefix, "Test run completed successfully") return nil } + +// WaitForDownloadCompleteWithRetry waits until no download is in progress +// using a fixed 30 second retry interval. This is useful for the test runner +// to wait for snapshot downloads to complete before proceeding. +func (s *TestRunnerService) WaitForDownloadCompleteWithRetry(ctx context.Context) error { + const retryInterval = 30 * time.Second + + for { + select { + case <-ctx.Done(): + return ctx.Err() + default: + inProgress, err := s.DownloadProgress.IsDownloadInProgress(ctx) + if err != nil { + return fmt.Errorf("error checking download progress: %w", err) + } + + if !inProgress { + return nil + } + + time.Sleep(retryInterval) + } + } +} diff --git a/internal/config/snapshot_checker_config.go b/internal/config/snapshot_checker_config.go new file mode 100644 index 0000000..39df813 --- /dev/null +++ b/internal/config/snapshot_checker_config.go @@ -0,0 +1,83 @@ +package config + +import ( + "clients-test/internal/application/domain" + "clients-test/internal/logger" + "flag" + "os" + "strings" +) + +var snapshotLogPrefix = "SnapshotCheckerConfig" + +// SnapshotCheckerAppConfig holds all application configuration for snapshot checker +type SnapshotCheckerAppConfig struct { + ExecutionClient string + Network string + CronIntervalSec int + RunOnce bool // If true, run once and exit (for testing) +} + +// DefaultCronIntervalSec is 6 hours in seconds +const DefaultCronIntervalSec = 6 * 60 * 60 + +// ParseSnapshotCheckerConfig parses CLI flags and environment variables into a SnapshotCheckerAppConfig +func ParseSnapshotCheckerConfig() SnapshotCheckerAppConfig { + // CLI flags + executionClient := flag.String("execution-client", "", "Execution client name (geth, nethermind, reth, besu, erigon). Required.") + network := flag.String("network", "hoodi", "Network name (e.g., hoodi). Default: hoodi") + cronIntervalSec := flag.Int("cron-interval", 0, "Interval between snapshot checks in seconds. Default: 21600 (6 hours)") + runOnce := flag.Bool("run-once", true, "Run once and exit (for testing). Default: false") + + flag.Parse() + + config := SnapshotCheckerAppConfig{ + ExecutionClient: strings.TrimSpace(strings.ToLower(getConfigValue(*executionClient, "EXECUTION_CLIENT", ""))), + Network: getConfigValue(*network, "NETWORK", "hoodi"), + CronIntervalSec: getCronIntervalConfigValueInt(*cronIntervalSec, "CRON_INTERVAL_SEC", DefaultCronIntervalSec), + RunOnce: *runOnce || os.Getenv("RUN_ONCE") == "true", + } + + return config +} + +// getCronIntervalConfigValueInt returns the flag value if set, otherwise falls back to the environment variable, then default +func getCronIntervalConfigValueInt(flagValue int, envName string, defaultValue int) int { + if flagValue != 0 { + return flagValue + } + if envValue := os.Getenv(envName); envValue != "" { + var result int + for _, c := range envValue { + if c >= '0' && c <= '9' { + result = result*10 + int(c-'0') + } + } + if result > 0 { + return result + } + } + return defaultValue +} + +// Validate checks that required configuration values are present and valid +func (c *SnapshotCheckerAppConfig) Validate() { + // Validate execution client is specified + if c.ExecutionClient == "" { + logger.FatalWithPrefix(snapshotLogPrefix, "Execution client is required. Valid clients: %v", domain.ValidExecutionClients) + } + + // Validate execution client is valid + if !domain.IsValidExecutionClient(c.ExecutionClient) { + logger.FatalWithPrefix(snapshotLogPrefix, "Invalid execution client '%s'. Valid clients: %v", c.ExecutionClient, domain.ValidExecutionClients) + } + + // Validate cron interval + if c.CronIntervalSec < 60 { + logger.WarnWithPrefix(snapshotLogPrefix, "Cron interval %d seconds is very short, using minimum of 60 seconds", c.CronIntervalSec) + c.CronIntervalSec = 60 + } + + logger.InfoWithPrefix(snapshotLogPrefix, "Configuration validated: network=%s, client=%s, interval=%ds, runOnce=%v", + c.Network, c.ExecutionClient, c.CronIntervalSec, c.RunOnce) +} diff --git a/internal/config/test_runner_config.go b/internal/config/test_runner_config.go new file mode 100644 index 0000000..907ec9f --- /dev/null +++ b/internal/config/test_runner_config.go @@ -0,0 +1,86 @@ +package config + +import ( + "clients-test/internal/adapters/apis/github" + "clients-test/internal/logger" + "flag" + "os" +) + +var logPrefix = "Config" + +// Config holds all application configuration +type Config struct { + IPFSGatewayURL string + IPFSHash string + ExecutionClient string // Optional: override execution client (e.g., geth, reth, nethermind) + ConsensusClient string // Optional: override consensus client (e.g., prysm, teku, nimbus, lodestar) + GitHub github.GitHubConfig +} + +// ParseConfig parses CLI flags and environment variables into a Config struct +func ParseConfig() Config { + // CLI flags + ipfsGatewayUrl := flag.String("ipfs-gateway-url", "", "IPFS gateway URL (required, or set IPFS_GATEWAY_URL env)") + ipfsHash := flag.String("ipfs-hash", "", "IPFS hash for the test package (required, or set IPFS_HASH env)") + + // Optional client override flags + executionClient := flag.String("execution-client", "", "Override execution client (geth, reth, nethermind, besu, erigon) (or set EXECUTION_CLIENT env)") + consensusClient := flag.String("consensus-client", "", "Override consensus client (prysm, teku, nimbus, lodestar) (or set CONSENSUS_CLIENT env)") + + // GitHub flags (optional, for PR commenting) + githubToken := flag.String("github-token", "", "GitHub token for PR commenting (or set GITHUB_TOKEN env)") + githubRepository := flag.String("github-repository", "", "GitHub repository in format owner/repo (or set GITHUB_REPOSITORY env)") + githubPRNumber := flag.String("github-pr-number", "", "Pull request number (or set GITHUB_PR_NUMBER env)") + githubRunID := flag.String("github-run-id", "", "GitHub Actions run ID (or set GITHUB_RUN_ID env)") + githubServerURL := flag.String("github-server-url", "", "GitHub server URL (or set GITHUB_SERVER_URL env)") + + flag.Parse() + + // Build config with flag values, falling back to environment variables + config := Config{ + IPFSGatewayURL: getConfigValue(*ipfsGatewayUrl, "IPFS_GATEWAY_URL", ""), + IPFSHash: getConfigValue(*ipfsHash, "IPFS_HASH", ""), + ExecutionClient: getConfigValue(*executionClient, "EXECUTION_CLIENT", ""), + ConsensusClient: getConfigValue(*consensusClient, "CONSENSUS_CLIENT", ""), + GitHub: github.ParseGitHubConfigFromEnv( + getConfigValue(*githubToken, "GITHUB_TOKEN", ""), + getConfigValue(*githubRepository, "GITHUB_REPOSITORY", ""), + getGitHubPRNumber(*githubPRNumber), + getConfigValue(*githubRunID, "GITHUB_RUN_ID", ""), + getGitHubServerURL(*githubServerURL), + ), + } + + return config +} + +// getGitHubPRNumber gets the PR number from flag or environment variables +func getGitHubPRNumber(flagValue string) string { + if flagValue != "" { + return flagValue + } + if prNumber := os.Getenv("GITHUB_PR_NUMBER"); prNumber != "" { + return prNumber + } + // Also check GITHUB_EVENT_NUMBER which is set in PR events + return os.Getenv("GITHUB_EVENT_NUMBER") +} + +// getGitHubServerURL gets the server URL with a default fallback +func getGitHubServerURL(flagValue string) string { + if flagValue != "" { + return flagValue + } + if serverURL := os.Getenv("GITHUB_SERVER_URL"); serverURL != "" { + return serverURL + } + return "https://github.com" +} + +// Validate checks that required configuration values are present +func (c *Config) Validate() { + if c.IPFSGatewayURL == "" || c.IPFSHash == "" { + logger.FatalWithPrefix(logPrefix, "IPFS gateway URL and hash are required. Set via --ipfs-gateway-url/--ipfs-hash flags or IPFS_GATEWAY_URL/IPFS_HASH environment variables.") + } +} diff --git a/internal/config/utils.go b/internal/config/utils.go new file mode 100644 index 0000000..b65d193 --- /dev/null +++ b/internal/config/utils.go @@ -0,0 +1,14 @@ +package config + +import "os" + +// getConfigValue returns the flag value if set, otherwise falls back to the environment variable, then default +func getConfigValue(flagValue, envName, defaultValue string) string { + if flagValue != "" { + return flagValue + } + if envValue := os.Getenv(envName); envValue != "" { + return envValue + } + return defaultValue +} diff --git a/internal/logger/logger.go b/internal/logger/logger.go index b9b35bb..6f5b5e6 100644 --- a/internal/logger/logger.go +++ b/internal/logger/logger.go @@ -50,7 +50,7 @@ func parseLogLevelFromEnv() LogLevel { case "FATAL": return FATAL default: - return INFO // Default to INFO if LOG_LEVEL is not set or invalid + return DEBUG // Default to DEBUG if LOG_LEVEL is not set or invalid } }