fix(nix): resolve hash race condition in parallel matrix jobs (#8995)

2 files changed, 173 insertions, 33 deletions

diff --git a/.github/workflows/nix-desktop.yml b/.github/workflows/nix-desktop.yml
index 01cfaed78..3d7c48031 100644
--- a/.github/workflows/nix-desktop.yml
+++ b/.github/workflows/nix-desktop.yml
@@ -9,6 +9,7 @@ on:
       - "nix/**"
       - "packages/app/**"
       - "packages/desktop/**"
+      - ".github/workflows/nix-desktop.yml"
   pull_request:
     paths:
       - "flake.nix"
@@ -16,6 +17,7 @@ on:
       - "nix/**"
       - "packages/app/**"
       - "packages/desktop/**"
+      - ".github/workflows/nix-desktop.yml"
   workflow_dispatch:
 
 jobs:
@@ -26,7 +28,7 @@ jobs:
         os:
           - blacksmith-4vcpu-ubuntu-2404
           - blacksmith-4vcpu-ubuntu-2404-arm
-          - macos-15
+          - macos-15-intel
           - macos-latest
     runs-on: ${{ matrix.os }}
     timeout-minutes: 60
diff --git a/.github/workflows/update-nix-hashes.yml b/.github/workflows/update-nix-hashes.yml
index 19373f748..f80a57d25 100644
--- a/.github/workflows/update-nix-hashes.yml
+++ b/.github/workflows/update-nix-hashes.yml
@@ -10,11 +10,13 @@ on:
       - "bun.lock"
       - "package.json"
       - "packages/*/package.json"
+      - ".github/workflows/update-nix-hashes.yml"
   pull_request:
     paths:
       - "bun.lock"
       - "package.json"
       - "packages/*/package.json"
+      - ".github/workflows/update-nix-hashes.yml"
 
 jobs:
   update-flake:
@@ -25,7 +27,7 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
           fetch-depth: 0
@@ -43,9 +45,9 @@ jobs:
       - name: Update ${{ env.TITLE }}
         run: |
           set -euo pipefail
-          echo "📦 Updating $TITLE..."
+          echo "Updating $TITLE..."
           nix flake update
-          echo "✅ $TITLE updated successfully"
+          echo "$TITLE updated successfully"
 
       - name: Commit ${{ env.TITLE }} changes
         env:
@@ -53,7 +55,7 @@ jobs:
         run: |
           set -euo pipefail
 
-          echo "🔍 Checking for changes in tracked files..."
+          echo "Checking for changes in tracked files..."
 
           summarize() {
             local status="$1"
@@ -71,29 +73,29 @@ jobs:
           FILES=(flake.lock flake.nix)
           STATUS="$(git status --short -- "${FILES[@]}" || true)"
           if [ -z "$STATUS" ]; then
-            echo "✅ No changes detected."
+            echo "No changes detected."
             summarize "no changes"
             exit 0
           fi
 
-          echo "📝 Changes detected:"
+          echo "Changes detected:"
           echo "$STATUS"
-          echo "🔗 Staging files..."
+          echo "Staging files..."
           git add "${FILES[@]}"
-          echo "💾 Committing changes..."
+          echo "Committing changes..."
           git commit -m "Update $TITLE"
-          echo "✅ Changes committed"
+          echo "Changes committed"
 
           BRANCH="${TARGET_BRANCH:-${GITHUB_REF_NAME}}"
-          echo "🌳 Pulling latest from branch: $BRANCH"
-          git pull --rebase origin "$BRANCH"
-          echo "🚀 Pushing changes to branch: $BRANCH"
+          echo "Pulling latest from branch: $BRANCH"
+          git pull --rebase --autostash origin "$BRANCH"
+          echo "Pushing changes to branch: $BRANCH"
           git push origin HEAD:"$BRANCH"
-          echo "✅ Changes pushed successfully"
+          echo "Changes pushed successfully"
 
           summarize "committed $(git rev-parse --short HEAD)"
 
-  update-node-modules-hash:
+  compute-node-modules-hash:
     needs: update-flake
     if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
     strategy:
@@ -111,11 +113,10 @@ jobs:
     runs-on: ${{ matrix.host }}
     env:
       SYSTEM: ${{ matrix.system }}
-      TITLE: node_modules hash (${{ matrix.system }})
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
           fetch-depth: 0
@@ -125,6 +126,104 @@ jobs:
       - name: Setup Nix
         uses: nixbuild/nix-quick-install-action@v34
 
+      - name: Compute node_modules hash
+        run: |
+          set -euo pipefail
+
+          DUMMY="sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
+          HASH_FILE="nix/hashes.json"
+          OUTPUT_FILE="hash-${SYSTEM}.txt"
+
+          export NIX_KEEP_OUTPUTS=1
+          export NIX_KEEP_DERIVATIONS=1
+
+          BUILD_LOG=$(mktemp)
+          TMP_JSON=$(mktemp)
+          trap 'rm -f "$BUILD_LOG" "$TMP_JSON"' EXIT
+
+          if [ ! -f "$HASH_FILE" ]; then
+            mkdir -p "$(dirname "$HASH_FILE")"
+            echo '{"nodeModules":{}}' > "$HASH_FILE"
+          fi
+
+          # Set dummy hash to force nix to rebuild and reveal correct hash
+          jq --arg system "$SYSTEM" --arg value "$DUMMY" \
+            '.nodeModules = (.nodeModules // {}) | .nodeModules[$system] = $value' "$HASH_FILE" > "$TMP_JSON"
+          mv "$TMP_JSON" "$HASH_FILE"
+
+          MODULES_ATTR=".#packages.${SYSTEM}.default.node_modules"
+          DRV_PATH="$(nix eval --raw "${MODULES_ATTR}.drvPath")"
+
+          echo "Building node_modules for ${SYSTEM} to discover correct hash..."
+          echo "Attempting to realize derivation: ${DRV_PATH}"
+          REALISE_OUT=$(nix-store --realise "$DRV_PATH" --keep-failed 2>&1 | tee "$BUILD_LOG" || true)
+
+          BUILD_PATH=$(echo "$REALISE_OUT" | grep "^/nix/store/" | head -n1 || true)
+          CORRECT_HASH=""
+
+          if [ -n "$BUILD_PATH" ] && [ -d "$BUILD_PATH" ]; then
+            echo "Realized node_modules output: $BUILD_PATH"
+            CORRECT_HASH=$(nix hash path --sri "$BUILD_PATH" 2>/dev/null || true)
+          fi
+
+          # Try to extract hash from build log
+          if [ -z "$CORRECT_HASH" ]; then
+            CORRECT_HASH="$(grep -E 'got:\s+sha256-[A-Za-z0-9+/=]+' "$BUILD_LOG" | awk '{print $2}' | head -n1 || true)"
+          fi
+
+          if [ -z "$CORRECT_HASH" ]; then
+            CORRECT_HASH="$(grep -A2 'hash mismatch' "$BUILD_LOG" | grep 'got:' | awk '{print $2}' | sed 's/sha256:/sha256-/' || true)"
+          fi
+
+          # Try to hash from kept failed build directory
+          if [ -z "$CORRECT_HASH" ]; then
+            KEPT_DIR=$(grep -oE "build directory.*'[^']+'" "$BUILD_LOG" | grep -oE "'/[^']+'" | tr -d "'" | head -n1 || true)
+            if [ -z "$KEPT_DIR" ]; then
+              KEPT_DIR=$(grep -oE '/nix/var/nix/builds/[^ ]+' "$BUILD_LOG" | head -n1 || true)
+            fi
+
+            if [ -n "$KEPT_DIR" ] && [ -d "$KEPT_DIR" ]; then
+              HASH_PATH="$KEPT_DIR"
+              [ -d "$KEPT_DIR/build" ] && HASH_PATH="$KEPT_DIR/build"
+
+              if [ -d "$HASH_PATH/node_modules" ]; then
+                CORRECT_HASH=$(nix hash path --sri "$HASH_PATH" 2>/dev/null || true)
+              fi
+            fi
+          fi
+
+          if [ -z "$CORRECT_HASH" ]; then
+            echo "Failed to determine correct node_modules hash for ${SYSTEM}."
+            cat "$BUILD_LOG"
+            exit 1
+          fi
+
+          echo "$CORRECT_HASH" > "$OUTPUT_FILE"
+          echo "Hash for ${SYSTEM}: $CORRECT_HASH"
+
+      - name: Upload hash artifact
+        uses: actions/upload-artifact@v6
+        with:
+          name: hash-${{ matrix.system }}
+          path: hash-${{ matrix.system }}.txt
+          retention-days: 1
+
+  commit-node-modules-hashes:
+    needs: compute-node-modules-hash
+    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
+    runs-on: blacksmith-4vcpu-ubuntu-2404
+    env:
+      TITLE: node_modules hashes
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          fetch-depth: 0
+          ref: ${{ github.head_ref || github.ref_name }}
+          repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }}
+
       - name: Configure git
         run: |
           git config --global user.email "[email protected]"
@@ -135,14 +234,57 @@ jobs:
           TARGET_BRANCH: ${{ github.head_ref || github.ref_name }}
         run: |
           BRANCH="${TARGET_BRANCH:-${GITHUB_REF_NAME}}"
-          git pull origin "$BRANCH"
+          git pull --rebase --autostash origin "$BRANCH"
 
-      - name: Update ${{ env.TITLE }}
+      - name: Download all hash artifacts
+        uses: actions/download-artifact@v7
+        with:
+          pattern: hash-*
+          merge-multiple: true
+
+      - name: Merge hashes into hashes.json
         run: |
           set -euo pipefail
-          echo "🔄 Updating $TITLE..."
-          nix/scripts/update-hashes.sh
-          echo "✅ $TITLE updated successfully"
+
+          HASH_FILE="nix/hashes.json"
+
+          if [ ! -f "$HASH_FILE" ]; then
+            mkdir -p "$(dirname "$HASH_FILE")"
+            echo '{"nodeModules":{}}' > "$HASH_FILE"
+          fi
+
+          echo "Merging hashes into ${HASH_FILE}..."
+
+          shopt -s nullglob
+          files=(hash-*.txt)
+          if [ ${#files[@]} -eq 0 ]; then
+            echo "No hash files found, nothing to update"
+            exit 0
+          fi
+
+          EXPECTED_SYSTEMS="x86_64-linux aarch64-linux x86_64-darwin aarch64-darwin"
+          for sys in $EXPECTED_SYSTEMS; do
+            if [ ! -f "hash-${sys}.txt" ]; then
+              echo "WARNING: Missing hash file for $sys"
+            fi
+          done
+
+          for f in "${files[@]}"; do
+            system="${f#hash-}"
+            system="${system%.txt}"
+            hash=$(cat "$f")
+            if [ -z "$hash" ]; then
+              echo "WARNING: Empty hash for $system, skipping"
+              continue
+            fi
+            echo "  $system: $hash"
+            jq --arg sys "$system" --arg h "$hash" \
+              '.nodeModules = (.nodeModules // {}) | .nodeModules[$sys] = $h' "$HASH_FILE" > "${HASH_FILE}.tmp"
+            mv "${HASH_FILE}.tmp" "$HASH_FILE"
+          done
+
+          echo "All hashes merged:"
+          cat "$HASH_FILE"
 
       - name: Commit ${{ env.TITLE }} changes
         env:
@@ -150,7 +292,8 @@ jobs:
         run: |
           set -euo pipefail
 
-          echo "🔍 Checking for changes in tracked files..."
+          HASH_FILE="nix/hashes.json"
+          echo "Checking for changes..."
 
           summarize() {
             local status="$1"
@@ -166,27 +309,22 @@ jobs:
             echo "" >> "$GITHUB_STEP_SUMMARY"
           }
 
-          FILES=(nix/hashes.json)
+          FILES=("$HASH_FILE")
           STATUS="$(git status --short -- "${FILES[@]}" || true)"
           if [ -z "$STATUS" ]; then
-            echo "✅ No changes detected."
+            echo "No changes detected."
             summarize "no changes"
             exit 0
           fi
 
-          echo "📝 Changes detected:"
+          echo "Changes detected:"
           echo "$STATUS"
-          echo "🔗 Staging files..."
           git add "${FILES[@]}"
-          echo "💾 Committing changes..."
           git commit -m "Update $TITLE"
-          echo "✅ Changes committed"
 
           BRANCH="${TARGET_BRANCH:-${GITHUB_REF_NAME}}"
-          echo "🌳 Pulling latest from branch: $BRANCH"
-          git pull --rebase origin "$BRANCH"
-          echo "🚀 Pushing changes to branch: $BRANCH"
+          git pull --rebase --autostash origin "$BRANCH"
           git push origin HEAD:"$BRANCH"
-          echo "✅ Changes pushed successfully"
+          echo "Changes pushed successfully"
 
           summarize "committed $(git rev-parse --short HEAD)"