feat(runner): Add ability to create cells without links and unify cell implementations #5492
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Deno Workflow | |
| on: | |
| push: | |
| branches: | |
| - main | |
| pull_request: | |
| branches: | |
| - main | |
| # Define the list of binaries we're building | |
| # This makes it easy to add more binaries in the future | |
| env: | |
| BINARIES: "toolshed bg-charm-service ct" | |
| jobs: | |
| build: | |
| name: "Test and Build" | |
| runs-on: ubuntu-24.04-32-core | |
| steps: | |
| - name: 📥 Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: 🦕 Setup Deno | |
| uses: ./.github/actions/deno-setup | |
| # Errors if `deno.lock` file was not committed with the current change | |
| - name: 🔍 Verify lock file & install dependencies | |
| run: deno install --frozen=true | |
| - name: 📥 Download Deno dependency binaries | |
| run: deno task initialize-db | |
| - name: 🔎 Type check codebase | |
| run: deno task check | |
| - name: 🔎 Check codebase formatting | |
| run: deno fmt --check | |
| - name: 🧹 Lint codebase | |
| run: deno lint | |
| # For deno-web-test browser tests | |
| # https://github.com/lino-levan/astral/blob/f5ef833b2c5bde3783564a6b925073d5d46bb4b8/README.md#no-usable-sandbox-with-user-namespace-cloning-enabled | |
| - name: 🛡️ Disable AppArmor for browser tests | |
| run: echo 0 | sudo tee /proc/sys/kernel/apparmor_restrict_unprivileged_userns | |
| - name: 🧪 Run parallel workspace tests | |
| run: deno task test | |
| - name: 🏗️ Build application binaries | |
| run: deno task build-binaries | |
| env: | |
| COMMIT_SHA: ${{ github.sha }} | |
| # Upload binaries as artifacts for the integration tests | |
| - name: 📤 Upload binaries for integration tests | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: common-binaries | |
| path: | | |
| ./dist/toolshed | |
| ./dist/bg-charm-service | |
| ./dist/ct | |
| package-integration-test: | |
| name: "Package Integration Tests" | |
| runs-on: ubuntu-latest | |
| needs: ["build"] | |
| environment: ci | |
| steps: | |
| - name: 📥 Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: 🦕 Setup Deno | |
| uses: ./.github/actions/deno-setup | |
| - name: 📥 Download built binaries | |
| uses: actions/download-artifact@v4 | |
| - name: 🚀 Start Toolshed server for testing | |
| run: | | |
| chmod +x ./common-binaries/toolshed | |
| CTTS_AI_LLM_ANTHROPIC_API_KEY=fake \ | |
| ./common-binaries/toolshed & | |
| # For Astral | |
| # https://github.com/lino-levan/astral/blob/f5ef833b2c5bde3783564a6b925073d5d46bb4b8/README.md#no-usable-sandbox-with-user-namespace-cloning-enabled | |
| - name: 🛡️ Disable AppArmor for browser tests | |
| run: echo 0 | sudo tee /proc/sys/kernel/apparmor_restrict_unprivileged_userns | |
| - name: 🧪 Run end-to-end runner integration tests | |
| working-directory: packages/runner | |
| run: | | |
| API_URL=http://localhost:8000/ \ | |
| deno task integration | |
| - name: 🧪 Run end-to-end shell integration tests | |
| working-directory: packages/shell | |
| run: | | |
| HEADLESS=1 \ | |
| API_URL=http://localhost:8000/ \ | |
| deno task integration | |
| - name: 🧪 Run background worker integration tests | |
| working-directory: packages/background-charm-service | |
| run: | | |
| HEADLESS=1 \ | |
| API_URL=http://localhost:8000/ \ | |
| deno task integration | |
| cli-integration-test: | |
| name: "CLI Integration Tests" | |
| runs-on: ubuntu-latest | |
| needs: ["build"] | |
| environment: ci | |
| steps: | |
| - name: 📥 Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: 🦕 Setup Deno | |
| uses: ./.github/actions/deno-setup | |
| - name: 📥 Download built binaries | |
| uses: actions/download-artifact@v4 | |
| - name: 🚀 Start Toolshed server for testing | |
| run: | | |
| chmod +x ./common-binaries/ct | |
| chmod +x ./common-binaries/toolshed | |
| # Set tools to path | |
| # Integration script needs `ct` | |
| echo "${{ github.workspace }}/common-binaries" >> $GITHUB_PATH | |
| ./common-binaries/toolshed & | |
| - name: 🧪 Run CLI integration tests | |
| working-directory: packages/cli | |
| run: | | |
| API_URL=http://localhost:8000 ./integration/integration.sh | |
| pattern-integration-test: | |
| name: "Pattern Integration Tests" | |
| runs-on: ubuntu-latest | |
| needs: ["build"] | |
| environment: ci | |
| steps: | |
| - name: 📥 Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: 🦕 Setup Deno | |
| uses: ./.github/actions/deno-setup | |
| - name: 📥 Download built binaries | |
| uses: actions/download-artifact@v4 | |
| - name: 🚀 Start Toolshed server for testing | |
| run: | | |
| chmod +x ./common-binaries/toolshed | |
| CTTS_AI_LLM_ANTHROPIC_API_KEY=fake \ | |
| ./common-binaries/toolshed & | |
| # For Astral | |
| # https://github.com/lino-levan/astral/blob/f5ef833b2c5bde3783564a6b925073d5d46bb4b8/README.md#no-usable-sandbox-with-user-namespace-cloning-enabled | |
| - name: 🛡️ Disable AppArmor for browser tests | |
| run: echo 0 | sudo tee /proc/sys/kernel/apparmor_restrict_unprivileged_userns | |
| - name: 🧩 Run end-to-end patterns integration tests | |
| working-directory: packages/patterns | |
| run: | | |
| HEADLESS=1 \ | |
| API_URL=http://localhost:8000/ \ | |
| deno task integration | |
| generated-patterns-integration-test: | |
| name: "Generated Patterns Integration Tests" | |
| runs-on: ubuntu-latest | |
| needs: ["build"] | |
| environment: ci | |
| steps: | |
| - name: 📥 Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: 🦕 Setup Deno | |
| uses: ./.github/actions/deno-setup | |
| - name: 🧪 Run generated patterns integration tests | |
| working-directory: packages/generated-patterns | |
| run: deno task integration | |
| attest-binaries: | |
| name: "Attest and Upload Binaries" | |
| if: github.ref == 'refs/heads/main' | |
| runs-on: ubuntu-24.04-32-core | |
| needs: [ | |
| "pattern-integration-test", | |
| "package-integration-test", | |
| "cli-integration-test", | |
| ] | |
| environment: ci | |
| permissions: | |
| id-token: write | |
| contents: read | |
| actions: read | |
| attestations: write | |
| steps: | |
| - name: 📥 Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: 📥 Download built binaries | |
| uses: actions/download-artifact@v4 | |
| - name: 🔐 Process & sign binaries | |
| run: | | |
| mkdir -p release | |
| mkdir -p signed | |
| mv ./common-binaries ./dist | |
| # Process toolshed binary | |
| echo "Processing binary: toolshed" | |
| sha256sum ./dist/toolshed > ./dist/toolshed.hash.txt | |
| TOOLSHED_HASH=$(cat ./dist/toolshed.hash.txt | awk '{print $1}') | |
| echo "toolshed hash: $TOOLSHED_HASH" | |
| echo "toolshed_hash=$TOOLSHED_HASH" >> $GITHUB_OUTPUT | |
| # Sign the toolshed binary | |
| openssl dgst -sha256 -sign <(echo "${{ secrets.ARTIFACT_SIGNING_KEY }}") -out ./dist/toolshed.sig ./dist/toolshed | |
| # Copy to signed directory | |
| cp ./dist/toolshed ./signed/ | |
| cp ./dist/toolshed.sig ./signed/ | |
| # Process bg-charm-service binary | |
| echo "Processing binary: bg-charm-service" | |
| sha256sum ./dist/bg-charm-service > ./dist/bg-charm-service.hash.txt | |
| BG_CHARM_SERVICE_HASH=$(cat ./dist/bg-charm-service.hash.txt | awk '{print $1}') | |
| echo "bg-charm-service hash: $BG_CHARM_SERVICE_HASH" | |
| echo "bg_charm_service_hash=$BG_CHARM_SERVICE_HASH" >> $GITHUB_OUTPUT | |
| # Sign the bg-charm-service binary | |
| openssl dgst -sha256 -sign <(echo "${{ secrets.ARTIFACT_SIGNING_KEY }}") -out ./dist/bg-charm-service.sig ./dist/bg-charm-service | |
| # Copy to signed directory | |
| cp ./dist/bg-charm-service ./signed/ | |
| cp ./dist/bg-charm-service.sig ./signed/ | |
| # Process ct binary | |
| echo "Processing binary: ct" | |
| sha256sum ./dist/ct > ./dist/ct.hash.txt | |
| CT_HASH=$(cat ./dist/ct.hash.txt | awk '{print $1}') | |
| echo "ct hash: $CT_HASH" | |
| echo "ct_hash=$CT_HASH" >> $GITHUB_OUTPUT | |
| # Sign the ct binary | |
| openssl dgst -sha256 -sign <(echo "${{ secrets.ARTIFACT_SIGNING_KEY }}") -out ./dist/ct.sig ./dist/ct | |
| # Copy to signed directory | |
| cp ./dist/ct ./signed/ | |
| cp ./dist/ct.sig ./signed/ | |
| # Create a single tarball with all binaries and signatures | |
| tar -czf release/labs-${{ github.sha }}.tar.gz -C signed . | |
| # Generate hash for the tarball | |
| sha256sum release/labs-${{ github.sha }}.tar.gz > release/labs-${{ github.sha }}.hash.txt | |
| TARBALL_HASH=$(cat release/labs-${{ github.sha }}.hash.txt | awk '{print $1}') | |
| echo "Tarball hash: $TARBALL_HASH" | |
| echo "tarball_hash=$TARBALL_HASH" >> $GITHUB_OUTPUT | |
| id: binary_processing | |
| - name: 📝 Generate attestation for toolshed binary | |
| id: attest_toolshed | |
| uses: actions/attest-build-provenance@v1 | |
| with: | |
| subject-name: ./dist/toolshed | |
| subject-digest: sha256:${{ steps.binary_processing.outputs.toolshed_hash }} | |
| - name: 📝 Generate attestation for bg-charm-service binary | |
| id: attest_bg_charm_service | |
| uses: actions/attest-build-provenance@v1 | |
| with: | |
| subject-name: ./dist/bg-charm-service | |
| subject-digest: sha256:${{ steps.binary_processing.outputs.bg_charm_service_hash }} | |
| - name: 📝 Generate attestation for ct binary | |
| id: attest_ct | |
| uses: actions/attest-build-provenance@v1 | |
| with: | |
| subject-name: ./dist/ct | |
| subject-digest: sha256:${{ steps.binary_processing.outputs.ct_hash }} | |
| - name: 📝 Generate attestation for tarball | |
| id: attest_tarball | |
| uses: actions/attest-build-provenance@v1 | |
| with: | |
| subject-name: https://storage.cloud.google.com/commontools-build-artifacts/workspace-artifacts/labs-${{ github.sha }}.tar.gz | |
| subject-digest: sha256:${{ steps.binary_processing.outputs.tarball_hash }} | |
| - name: 📤 Upload attestations as artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: binary_attestations | |
| if-no-files-found: error | |
| path: | | |
| ${{ steps.attest_toolshed.outputs.bundle-path }} | |
| ${{ steps.attest_bg_charm_service.outputs.bundle-path }} | |
| ${{ steps.attest_ct.outputs.bundle-path }} | |
| ${{ steps.attest_tarball.outputs.bundle-path }} | |
| - name: 🔍 Verify binary attestations | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| # Verify tarball attestation | |
| echo "::group::Tarball attestation details" | |
| gh attestation verify release/labs-${{ github.sha }}.tar.gz -R ${{ github.repository }} --format json | jq | |
| echo "::endgroup::" | |
| if [ $? -eq 0 ]; then | |
| echo -e "\033[32m✓ Tarball attestation verified successfully\033[0m" | |
| else | |
| echo -e "\033[31m✗ Tarball attestation verification failed\033[0m" | |
| exit 1 | |
| fi | |
| # Verify toolshed binary attestation | |
| echo "::group::toolshed attestation details" | |
| gh attestation verify ./dist/toolshed -R ${{ github.repository }} --format json | jq | |
| echo "::endgroup::" | |
| if [ $? -eq 0 ]; then | |
| echo -e "\033[32m✓ toolshed attestation verified successfully\033[0m" | |
| else | |
| echo -e "\033[31m✗ toolshed attestation verification failed\033[0m" | |
| exit 1 | |
| fi | |
| # Verify bg-charm-service binary attestation | |
| echo "::group::bg-charm-service attestation details" | |
| gh attestation verify ./dist/bg-charm-service -R ${{ github.repository }} --format json | jq | |
| echo "::endgroup::" | |
| if [ $? -eq 0 ]; then | |
| echo -e "\033[32m✓ bg-charm-service attestation verified successfully\033[0m" | |
| else | |
| echo -e "\033[31m✗ bg-charm-service attestation verification failed\033[0m" | |
| exit 1 | |
| fi | |
| # Verify ct binary attestation | |
| echo "::group::ct attestation details" | |
| gh attestation verify ./dist/ct -R ${{ github.repository }} --format json | jq | |
| echo "::endgroup::" | |
| if [ $? -eq 0 ]; then | |
| echo -e "\033[32m✓ ct attestation verified successfully\033[0m" | |
| else | |
| echo -e "\033[31m✗ ct attestation verification failed\033[0m" | |
| exit 1 | |
| fi | |
| - name: 🔑 Authenticate to Google Cloud | |
| uses: google-github-actions/auth@v1 | |
| with: | |
| credentials_json: ${{ secrets.GCP_SA_KEY }} | |
| - name: ⚙️ Setup Google Cloud SDK | |
| uses: google-github-actions/setup-gcloud@v1 | |
| - name: 🚀 Upload artifacts to Google Cloud Storage | |
| run: | | |
| gsutil cp release/labs-${{ github.sha }}.tar.gz gs://commontools-build-artifacts/workspace-artifacts/ | |
| gsutil cp release/labs-${{ github.sha }}.hash.txt gs://commontools-build-artifacts/workspace-artifacts/ | |
| # Print clickable links to the uploaded files | |
| echo "::group::📦 Artifact Links" | |
| echo "Tarball URL: https://storage.cloud.google.com/commontools-build-artifacts/workspace-artifacts/labs-${{ github.sha }}.tar.gz" | |
| echo "Hash URL: https://storage.cloud.google.com/commontools-build-artifacts/workspace-artifacts/labs-${{ github.sha }}.hash.txt" | |
| echo "::endgroup::" | |
| # Automatic deployment to staging (toolshed) | |
| deploy-toolshed: | |
| name: "Deploy to Toolshed (Staging)" | |
| if: github.ref == 'refs/heads/main' | |
| needs: ["attest-binaries"] | |
| runs-on: ubuntu-latest | |
| environment: toolshed | |
| steps: | |
| - name: 📥 Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: 🦕 Setup Deno | |
| uses: ./.github/actions/deno-setup | |
| with: | |
| cache: false | |
| - name: 🔽 Pre-download Sentry CLI | |
| run: | | |
| echo "::group::Downloading Sentry CLI" | |
| deno run --allow-all npm:@sentry/cli --version | |
| echo "::endgroup::" | |
| - name: 📊 Create Toolshed server Sentry release | |
| run: | | |
| # Create a release with version based on commit SHA | |
| deno run --allow-all npm:@sentry/cli releases new ${{ github.sha }} | |
| # Associate commits with the release | |
| deno run --allow-all npm:@sentry/cli releases set-commits ${{ github.sha }} --auto | |
| # Finalize the release | |
| deno run --allow-all npm:@sentry/cli releases finalize ${{ github.sha }} | |
| env: | |
| SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }} | |
| SENTRY_ORG: ${{ vars.SENTRY_ORG }} | |
| SENTRY_PROJECT: ${{ vars.SENTRY_TOOLSHED_PROJECT }} | |
| - name: 🚀 Deploy application to Toolshed (Staging) | |
| uses: appleboy/ssh-action@master | |
| with: | |
| host: ${{ secrets.BASTION_HOST }} | |
| username: bastion | |
| key: ${{ secrets.BASTION_SSH_PRIVATE_KEY }} | |
| script: /opt/ct/deploy.sh ${{ vars.DEPLOYMENT_ENVIRONMENT }} ${{ github.sha }} | |
| # Post-deployment patterns test (runs after deployment, so this will NOT block deployments.) | |
| post-deploy-patterns-test: | |
| name: "Toolshed Post-Deploy Patterns Test" | |
| if: github.ref == 'refs/heads/main' | |
| needs: ["deploy-toolshed"] | |
| runs-on: ubuntu-latest | |
| environment: toolshed | |
| continue-on-error: true # Don't fail the workflow if tests fail | |
| steps: | |
| - name: 🧩 Run post-deploy Patterns integration tests against Toolshed (Staging) | |
| id: run_tests | |
| continue-on-error: true | |
| uses: appleboy/ssh-action@master | |
| with: | |
| host: ${{ secrets.BASTION_HOST }} | |
| username: bastion | |
| key: ${{ secrets.BASTION_SSH_PRIVATE_KEY }} | |
| command_timeout: 10m | |
| script: | | |
| /opt/ct/run-pattern-tests-against-toolshed.sh ${{ github.sha }} \ | |
| https://toolshed.saga-castor.ts.net \ | |
| https://toolshed.saga-castor.ts.net | |
| - name: 📥 Retrieve test logs on failure | |
| if: steps.run_tests.outcome == 'failure' | |
| uses: appleboy/ssh-action@master | |
| with: | |
| host: ${{ secrets.BASTION_HOST }} | |
| username: bastion | |
| key: ${{ secrets.BASTION_SSH_PRIVATE_KEY }} | |
| script: | | |
| LOG_FILE="/tmp/patterns-test-${{ github.sha }}.log" | |
| if [ -f "$LOG_FILE" ]; then | |
| echo "📋 Test failed - showing last 500 lines..." | |
| echo "=========================================" | |
| tail -n 500 "$LOG_FILE" | |
| echo "=========================================" | |
| echo "Full logs will be available as artifact" | |
| else | |
| echo "⚠️ Log file not found at $LOG_FILE" | |
| fi | |
| - name: 📦 Download logs from bastion | |
| if: always() | |
| uses: nicklasfrahm/scp-action@main | |
| with: | |
| direction: download | |
| host: ${{ secrets.BASTION_HOST }} | |
| username: bastion | |
| key: ${{ secrets.BASTION_SSH_PRIVATE_KEY }} | |
| insecure_ignore_fingerprint: true | |
| source: /tmp/patterns-test-${{ github.sha }}.log | |
| target: patterns-test-${{ github.sha }}.log | |
| - name: 📤 Upload test logs as artifact | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: patterns-test-logs-${{ github.sha }} | |
| path: patterns-test-${{ github.sha }}.log | |
| retention-days: 7 | |
| if-no-files-found: warn | |
| - name: 📊 Report test status | |
| if: always() | |
| run: | | |
| if [ "${{ steps.run_tests.outcome }}" == "success" ]; then | |
| echo "✅ Pattern integration tests PASSED" | |
| else | |
| echo "❌ Pattern integration tests FAILED" | |
| echo "📥 Check the artifacts tab for full logs" | |
| fi | |
| exit ${{ steps.run_tests.outcome == 'success' && '0' || '1' }} |