diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 3e89fb3..4797379 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -34,7 +34,7 @@ jobs: run: echo "CI_IMAGE=${{ env.CI_IMAGE }}" >> $GITHUB_OUTPUT check-fmt: - runs-on: ubuntu-latest + runs-on: parity-default timeout-minutes: 20 needs: [set-image] container: @@ -57,7 +57,6 @@ jobs: check: name: Cargo check - # TODO: replace `parity-default` with `ubuntu-latest` when the repo becomes public. runs-on: parity-default needs: [set-image, check-fmt] container: @@ -77,7 +76,6 @@ jobs: check-benchmarking: name: Cargo check (benchmarking) - # TODO: replace `parity-default` with `ubuntu-latest` when the repo becomes public. runs-on: parity-default needs: [set-image, check] container: @@ -98,7 +96,6 @@ jobs: clippy: name: Cargo clippy - # TODO: replace `parity-default` with `ubuntu-latest` when the repo becomes public. runs-on: parity-default needs: [set-image, check] container: @@ -123,7 +120,6 @@ jobs: test: name: Test - # TODO: replace `parity-default` with `ubuntu-latest` when the repo becomes public. runs-on: parity-default timeout-minutes: 60 needs: [set-image, check] diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml new file mode 100644 index 0000000..0b3e59b --- /dev/null +++ b/.github/workflows/integration-test.yml @@ -0,0 +1,80 @@ +name: Integration Tests + +# Controls when the action will run. +on: + # Triggers the workflow on push or pull request events but only for the master branch + push: + branches: [main] + pull_request: + branches: [main] + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +# Cancel previous runs +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +# Parity CI image to use +# Common variable is defined in the workflow +# Repo env variable doesn't work for PRs from forks +env: + CI_IMAGE: "paritytech/ci-unified:bullseye-1.88.0-2025-06-27-v202507112050" + NODE_VERSION: 22 + +jobs: + set-image: + # This workaround sets the container image for each job using 'set-image' job output. + # env variables don't work for PRs from forks, so we need to use outputs. + runs-on: ubuntu-latest + outputs: + CI_IMAGE: ${{ steps.set_image.outputs.CI_IMAGE }} + steps: + - id: set_image + run: echo "CI_IMAGE=${{ env.CI_IMAGE }}" >> $GITHUB_OUTPUT + + integration-tests: + name: Integration Tests + runs-on: parity-default + timeout-minutes: 60 + needs: [ set-image ] + container: + image: ${{ needs.set-image.outputs.CI_IMAGE }} + + steps: + - name: Checkout sources + uses: actions/checkout@v4 + + - name: Rust cache + uses: Swatinem/rust-cache@v2 + with: + cache-on-failure: true + cache-all-crates: true + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: ${{ env.NODE_VERSION }} + cache: 'npm' + cache-dependency-path: examples/package.json + + - name: Install just + run: cargo install just --locked || true + + - name: Download zombienet + run: | + curl -L \ + -H "Authorization: token ${{ github.token }}" \ + -o zombienet-linux-x64 \ + "https://github.com/paritytech/zombienet/releases/download/v1.3.138/zombienet-linux-x64" + chmod +x zombienet-linux-x64 + echo "ZOMBIENET_BINARY=$GITHUB_WORKSPACE/zombienet-linux-x64" >> $GITHUB_ENV + + - name: Run authorize and store (PAPI, smoldot) + working-directory: examples + run: just run-authorize-and-store "smoldot" + + - name: Run authorize and store (PAPI, RPC node) + working-directory: examples + run: just run-authorize-and-store "ws" diff --git a/examples/README.md b/examples/README.md index fe0f05a..499a406 100644 --- a/examples/README.md +++ b/examples/README.md @@ -75,7 +75,7 @@ tar -xvzf kubo_v0.38.1_darwin-arm64.tar.gz #### Use Docker -* Use `172.17.0.1` or `host.docker.internal` for swarm connections +* Use `host.docker.internal` (macOS/Windows) or `172.17.0.1` (Linux) for swarm connections ```shell docker pull ipfs/kubo:latest @@ -107,7 +107,11 @@ POLKADOT_BULLETIN_BINARY_PATH=./target/release/polkadot-bulletin-chain \ ``` ```shell -# Uses Docker (replace 127.0.0.1 with 172.17.0.1) +# Uses Docker on macOS/Windows (use dns4/host.docker.internal) +docker exec -it ipfs-node ipfs swarm connect /dns4/host.docker.internal/tcp/10001/ws/p2p/12D3KooWQCkBm1BYtkHpocxCwMgR8yjitEeHGx8spzcDLGt2gkBm +docker exec -it ipfs-node ipfs swarm connect /dns4/host.docker.internal/tcp/12347/ws/p2p/12D3KooWRkZhiRhsqmrQ28rt73K7V3aCBpqKrLGSXmZ99PTcTZby + +# Uses Docker on Linux (use ip4/172.17.0.1) docker exec -it ipfs-node ipfs swarm connect /ip4/172.17.0.1/tcp/10001/ws/p2p/12D3KooWQCkBm1BYtkHpocxCwMgR8yjitEeHGx8spzcDLGt2gkBm docker exec -it ipfs-node ipfs swarm connect /ip4/172.17.0.1/tcp/12347/ws/p2p/12D3KooWRkZhiRhsqmrQ28rt73K7V3aCBpqKrLGSXmZ99PTcTZby ``` diff --git a/examples/justfile b/examples/justfile index 7c0bdd8..96ea8ee 100644 --- a/examples/justfile +++ b/examples/justfile @@ -28,133 +28,171 @@ _ipfs-cmd: exit 1 fi -# Initialize IPFS if not already initialized -ipfs-init: +# Start start solo chain with zombienet in background +bulletin-solo-zombienet-start: #!/usr/bin/env bash set -e - IPFS_CMD=$(just _ipfs-cmd) - echo "๐Ÿ”ง Using IPFS: $IPFS_CMD" - if [ ! -d ~/.ipfs ]; then - echo "๐Ÿ“ฆ Initializing IPFS..." - $IPFS_CMD init + mkdir -p {{ PID_DIR }} + + echo "โšก Starting Bulletin chain with zombienet..." + + # Find zombienet binary (use env var if set, otherwise search) + if [ -n "$ZOMBIENET_BINARY" ]; then + ZOMBIENET_BIN="$ZOMBIENET_BINARY" else - echo "โœ… IPFS already initialized" + ZOMBIENET_BIN=$(ls ../zombienet-*-* 2>/dev/null | head -n 1) + fi + + if [ -z "$ZOMBIENET_BIN" ] || [ ! -f "$ZOMBIENET_BIN" ]; then + echo "โŒ Error: Zombienet binary not found" + echo " Tried: $ZOMBIENET_BIN" + echo " Set ZOMBIENET_BINARY environment variable or ensure zombienet-* exists in parent directory" + exit 1 + fi + + echo " Using zombienet: $ZOMBIENET_BIN" + + # Get absolute paths for bulletin binary and config + BULLETIN_BINARY=$(cd .. && pwd)/target/release/polkadot-bulletin-chain + ZOMBIENET_CONFIG=$(cd .. && pwd)/zombienet/bulletin-polkadot-local.toml + + POLKADOT_BULLETIN_BINARY_PATH=$BULLETIN_BINARY $ZOMBIENET_BIN -p native spawn $ZOMBIENET_CONFIG > /tmp/zombienet.log 2>&1 & + ZOMBIENET_PID=$! + echo $ZOMBIENET_PID > {{ PID_DIR }}/zombienet.pid + echo " Zombienet PID: $ZOMBIENET_PID" + echo " Log: /tmp/zombienet.log" + echo " Waiting for chain to start (15 seconds)..." + sleep 15 + +# Check if Docker is available and running +_check-docker: + #!/usr/bin/env bash + if ! command -v docker &> /dev/null; then + echo "โŒ Error: Docker is not installed." + echo " Please install Docker Desktop from: https://www.docker.com/products/docker-desktop" + exit 1 + fi + + if ! docker ps &> /dev/null; then + echo "โŒ Error: Docker is not running." + echo " Please start Docker Desktop and try again." + echo " On macOS: Open Docker Desktop from Applications" + exit 1 fi -# Start IPFS daemon in background -ipfs-start: +# Start IPFS Docker container +ipfs-start: _check-docker #!/usr/bin/env bash set -e mkdir -p {{ PID_DIR }} - IPFS_CMD=$(just _ipfs-cmd) - - echo "๐Ÿ”ง Using IPFS: $IPFS_CMD" - echo "๐Ÿ“ก Starting IPFS daemon..." - $IPFS_CMD daemon > /tmp/ipfs-daemon.log 2>&1 & - echo $! > {{ PID_DIR }}/ipfs.pid - echo " IPFS PID: $!" - echo " Log: /tmp/ipfs-daemon.log" - sleep 2 + + echo "๐Ÿณ Starting IPFS Docker container..." + + # Pull latest kubo image if not present + docker pull ipfs/kubo:latest + + # Determine network mode based on OS + if [[ "$OSTYPE" == "darwin"* ]]; then + # macOS - use port mapping + NETWORK_ARGS="-p 4001:4001 -p 8080:8080 -p 5001:5001" + else + # Linux (including CI) - use host networking for direct access + NETWORK_ARGS="--network host -p 4001:4001 -p 8080:8080 -p 5001:5001" + fi + + # Start Docker container + docker run -d --name ipfs-node -v ipfs-data:/data/ipfs $NETWORK_ARGS ipfs/kubo:latest + echo " Container: ipfs-node" + echo " Waiting for container to start..." + sleep 5 + + # Store container name in PID directory for cleanup + echo "ipfs-node" > {{ PID_DIR }}/ipfs-docker.container -# Connect to IPFS nodes +# Connect to IPFS nodes using Docker ipfs-connect: #!/usr/bin/env bash set -e - IPFS_CMD=$(just _ipfs-cmd) - echo "๐Ÿ”ง Using IPFS: $IPFS_CMD" - echo "๐Ÿ”— Connecting IPFS nodes..." - $IPFS_CMD swarm connect /ip4/127.0.0.1/tcp/12347/ws/p2p/12D3KooWRkZhiRhsqmrQ28rt73K7V3aCBpqKrLGSXmZ99PTcTZby || true - $IPFS_CMD swarm connect /ip4/127.0.0.1/tcp/10001/ws/p2p/12D3KooWQCkBm1BYtkHpocxCwMgR8yjitEeHGx8spzcDLGt2gkBm || true - -# Shutdown IPFS daemon -ipfs-shutdown: - #!/usr/bin/env bash - IPFS_CMD=$(just _ipfs-cmd) + echo "๐Ÿ”— Connecting IPFS nodes (Docker)..." - echo "๐Ÿ”ง Using IPFS: $IPFS_CMD" - echo "๐Ÿ›‘ Shutting down IPFS daemon..." - $IPFS_CMD shutdown 2>/dev/null || echo " โš  IPFS not running or already stopped" + # Detect the correct host and protocol for Docker + if [[ "$OSTYPE" == "darwin"* ]]; then + # macOS - use dns4/host.docker.internal + PROTOCOL="dns4" + DOCKER_HOST="host.docker.internal" + else + # Linux (with host networking) - use ip4/127.0.0.1 + PROTOCOL="ip4" + DOCKER_HOST="127.0.0.1" + fi - # Clean up PID file if it exists - [ -f "{{ PID_DIR }}/ipfs.pid" ] && rm "{{ PID_DIR }}/ipfs.pid" && echo " โœ“ Cleaned up PID file" || true + echo " Using Docker host: /$PROTOCOL/$DOCKER_HOST" + docker exec ipfs-node ipfs swarm connect /$PROTOCOL/$DOCKER_HOST/tcp/10001/ws/p2p/12D3KooWQCkBm1BYtkHpocxCwMgR8yjitEeHGx8spzcDLGt2gkBm || true + docker exec ipfs-node ipfs swarm connect /$PROTOCOL/$DOCKER_HOST/tcp/12347/ws/p2p/12D3KooWRkZhiRhsqmrQ28rt73K7V3aCBpqKrLGSXmZ99PTcTZby || true -# Start start solo chain with zombienet in background -bulletin-solo-zombienet-start: +# Shutdown IPFS Docker container +ipfs-shutdown: #!/usr/bin/env bash - set -e - mkdir -p {{ PID_DIR }} + echo "๐Ÿ›‘ Shutting down IPFS Docker container..." + docker stop ipfs-node 2>/dev/null || echo " โš  Container not running or already stopped" + docker rm ipfs-node 2>/dev/null || echo " โš  Container not found or already removed" - echo "โšก Starting Bulletin chain with zombienet..." - POLKADOT_BULLETIN_BINARY_PATH=../target/release/polkadot-bulletin-chain ../$(ls ../zombienet-*-*) -p native spawn ../zombienet/bulletin-polkadot-local.toml > /tmp/zombienet.log 2>&1 & - ZOMBIENET_PID=$! - echo $ZOMBIENET_PID > {{ PID_DIR }}/zombienet.pid - echo " Zombienet PID: $ZOMBIENET_PID" - echo " Log: /tmp/zombienet.log" - echo " Waiting for chain to start (15 seconds)..." - sleep 15 + # Clean up container file if it exists + [ -f "{{ PID_DIR }}/ipfs-docker.container" ] && rm "{{ PID_DIR }}/ipfs-docker.container" && echo " โœ“ Cleaned up container file" || true -# Start IPFS reconnect script in background +# Start IPFS reconnect script in background (Docker mode) ipfs-reconnect-start: #!/usr/bin/env bash set -e mkdir -p {{ PID_DIR }} - echo "๐Ÿ”„ Starting IPFS reconnect script..." - ./scripts/ipfs-reconnect-solo.sh > /tmp/ipfs-reconnect.log 2>&1 & + echo "๐Ÿ”„ Starting IPFS reconnect script (Docker mode)..." + ./scripts/ipfs-reconnect-solo.sh docker > /tmp/ipfs-reconnect-docker.log 2>&1 & RECONNECT_PID=$! - echo $RECONNECT_PID > {{ PID_DIR }}/ipfs-reconnect.pid + echo $RECONNECT_PID > {{ PID_DIR }}/ipfs-reconnect-docker.pid echo " Reconnect PID: $RECONNECT_PID" - echo " Log: /tmp/ipfs-reconnect.log" + echo " Log: /tmp/ipfs-reconnect-docker.log" sleep 2 -# Generate PAPI descriptors -papi-generate: +# Setup all services using Docker for IPFS +setup-services: #!/usr/bin/env bash set -e - echo "๐Ÿ”ง Generating PAPI descriptors..." - npm run papi:generate - -# Setup all services needed for testing (IPFS, zombienet, IPFS reconnect, PAPI) -# Parameters: api=[papi|pjs] - choose between Polkadot API (papi) or Polkadot JS (pjs) -setup-services api="papi": - #!/usr/bin/env bash - set -e - - API_TYPE="{{ api }}" - - echo "๐Ÿ”ง Tearing down services if they are running..." + echo "๐Ÿ”ง Tearing down Docker services if they are running..." just ipfs-shutdown - echo "๐Ÿ”ง Setting up services for $API_TYPE..." - - # Initialize IPFS - just ipfs-init + echo "๐Ÿณ Setting up services with Docker IPFS..." - # Start services + # Start services with Docker just ipfs-start just bulletin-solo-zombienet-start + + # Wait a bit longer and verify zombienet is running + echo "๐Ÿ” Verifying zombienet is ready..." + sleep 10 + echo " Zombienet process check:" + ps aux | grep zombienet | grep -v grep || echo " โš  No zombienet process found" + echo " Checking zombienet log:" + tail -20 /tmp/zombienet.log || echo " โš  Could not read zombienet log" + just ipfs-connect just ipfs-reconnect-start + just papi-generate - # Generate PAPI descriptors if needed - if [ "$API_TYPE" == "papi" ]; then - just papi-generate - fi - - echo "โœ… Services setup complete" + echo "โœ… Services setup complete (Docker mode)" -# Stop all running services (IPFS, zombienet, reconnect script) +# Stop all Docker services teardown-services: #!/usr/bin/env bash - echo "๐Ÿงน Stopping all services..." + echo "๐Ÿงน Stopping all Docker services..." - # Stop services by reading PIDs from files + # Stop reconnect script if [ -d {{ PID_DIR }} ]; then for pidfile in {{ PID_DIR }}/*.pid; do if [ -f "$pidfile" ]; then @@ -168,39 +206,39 @@ teardown-services: rm "$pidfile" fi done + fi + + # Stop Docker container + just ipfs-shutdown + + # Clean up PID directory + if [ -d {{ PID_DIR }} ]; then rmdir {{ PID_DIR }} 2>/dev/null || true - else - echo " No running services found" fi - echo "โœ… Services stopped" + echo "โœ… Docker services stopped" -# Test the complete workflow (builds, starts services, runs example, shuts down services) -# Parameters: api=[papi|pjs] - choose between Polkadot API (papi) or Polkadot JS (pjs) -run-authorize-and-store api="papi": build npm-install +# Run authorize and store example with Docker IPFS +# Parameters: +# mode - Connection mode: "ws" (WebSocket RPC node) or "smoldot" (light client) +run-authorize-and-store mode="ws": build npm-install #!/usr/bin/env bash set -e - - API_TYPE="{{ api }}" - - if [[ "$API_TYPE" != "papi" && "$API_TYPE" != "pjs" ]]; then - echo "โŒ Error: api parameter must be 'papi' or 'pjs'" + + if [ "{{ mode }}" = "smoldot" ]; then + echo "๐Ÿš€ Starting authorize and store workflow test (mode: smoldot)..." + SCRIPT_NAME="authorize_and_store_papi_smoldot.js" + elif [ "{{ mode }}" = "ws" ]; then + echo "๐Ÿš€ Starting authorize and store workflow test (mode: ws)..." + SCRIPT_NAME="authorize_and_store_papi.js" + else + echo "โŒ Error: Invalid mode '{{ mode }}'. Must be 'ws' or 'smoldot'" exit 1 fi - - echo "๐Ÿš€ Starting $API_TYPE workflow test..." echo "" - # Setup all services - just setup-services $API_TYPE - - # Run the example - if [ "$API_TYPE" == "papi" ]; then - EXAMPLE_FILE="authorize_and_store_papi.js" - else - EXAMPLE_FILE="authorize_and_store.js" - fi - node $EXAMPLE_FILE + just setup-services + node $SCRIPT_NAME EXAMPLE_EXIT=$? echo "" @@ -211,24 +249,6 @@ run-authorize-and-store api="papi": build npm-install echo "โŒ Example failed with exit code $EXAMPLE_EXIT" fi - echo "" - echo "๐Ÿ“ Logs available at:" - echo " /tmp/ipfs-daemon.log" - echo " /tmp/zombienet.log" - echo " /tmp/ipfs-reconnect.log" - - # Clean up background processes - just teardown-services - + just teardown-services exit $EXAMPLE_EXIT -run-authorize-and-store-smoldot: - #!/usr/bin/env bash - set -e - - just setup-services - node authorize_and_store_papi_smoldot.js - EXAMPLE_EXIT=$? - just teardown-services - - exit $EXAMPLE_EXIT \ No newline at end of file diff --git a/examples/package.json b/examples/package.json index 2b7de3c..59846fb 100644 --- a/examples/package.json +++ b/examples/package.json @@ -15,7 +15,8 @@ "ipfs-http-client": "^60.0.1", "multiformats": "^13.4.1", "polkadot-api": "^1.20.6", - "smoldot": "^2.0.40" + "smoldot": "^2.0.40", + "ws": "^8.18.0" }, "devDependencies": { "@polkadot-api/cli": "^0.13.3"