fix(test): stabilize scheduled E2E AutoTest runs (#1730)

* fix(test): stabilize scheduled E2E runs

Use deterministic disk checks for save steps, skip LLM verification for the no-op navigation assertion, and avoid unnecessary apt traffic when Xvfb is already installed.


* fix(test): use Java pre-release for navigation test

The Go to Super Implementation plan covers vscode-java#4438, which is present in stable 1.55.0 and fixed in the pre-release used by scheduled runs.

Author: wenytang-ms

.github/workflows/e2e-autotest.yml

@@ -153,12 +153,33 @@ jobs:
         if: runner.os == 'Linux'
         shell: bash
         run: |
-          sudo apt-get update
-          sudo apt-get install -y xvfb
-          Xvfb :99 -screen 0 1920x1080x24 &
+          if ! command -v Xvfb >/dev/null 2>&1; then
+            echo "Xvfb is not preinstalled; installing it with bounded apt operations."
+            apt_options=(
+              -o Acquire::Retries=3
+              -o Acquire::http::Timeout=15
+              -o Acquire::https::Timeout=15
+            )
+            if ! sudo timeout 120s apt-get "${apt_options[@]}" update; then
+              echo "::error::Timed out while refreshing apt metadata for Xvfb."
+              exit 1
+            fi
+            if ! sudo timeout 120s apt-get "${apt_options[@]}" install -y --no-install-recommends xvfb; then
+              echo "::error::Timed out while installing Xvfb."
+              exit 1
+            fi
+          fi
+
+          Xvfb :99 -screen 0 1920x1080x24 >"$RUNNER_TEMP/xvfb.log" 2>&1 &
+          xvfb_pid=$!
           echo "DISPLAY=:99" >> "$GITHUB_ENV"
           # Give Xvfb a moment to start before the autotest CLI launches VS Code.
           sleep 2
+          if ! kill -0 "$xvfb_pid" 2>/dev/null; then
+            cat "$RUNNER_TEMP/xvfb.log"
+            echo "::error::Xvfb failed to start."
+            exit 1
+          fi
 
       - name: Download vscode-java-pack VSIX (from branch)
         if: ${{ github.event_name != 'schedule' }}

test-plans/java-go-to-super-implementation.yaml

@@ -25,6 +25,9 @@ setup:
   extension: "redhat.java"
   extensions:
     - "vscjava.vscode-java-pack"
+  # vscode-java 1.55.0 stable contains the #4438 regression covered by this
+  # plan; use the pre-release carrying the fix on PR runs as schedules do.
+  preRelease: true
   vscodeVersion: "stable"
   workspace: "../test-fixtures/super-implementation"
   timeout: 300
@@ -82,3 +85,6 @@ steps:
       fileName: "Base.java"
       contains: "Hello from Base"
     timeout: 15
+    # Navigation completed in the previous step, so this settle-and-assert
+    # step is visually unchanged by design. verifyEditor is authoritative.
+    skipLlmVerify: true

test-plans/java-gradle.yaml

@@ -72,4 +72,10 @@ steps:
 
   - id: "save-file"
     action: "saveFile"
-    verify: "File saved"
+    verify: "Test1.java is saved to disk"
+    verifyFile:
+      path: "~/project1/src/main/java/project1/Test1.java"
+      contains: "// gradle test marker"
+    # The disk assertion proves saveFile persisted the in-memory edit. The
+    # screenshot-only check can misread the small tab dirty-dot transition.
+    skipLlmVerify: true

test-plans/java-maven.yaml

@@ -70,4 +70,10 @@ steps:
   # 2e. Save file
   - id: "save-file"
     action: "saveFile"
-    verify: "File saved"
+    verify: "Foo.java is saved to disk"
+    verifyFile:
+      path: "~/src/main/java/java/Foo.java"
+      contains: "// autotest marker"
+    # The disk assertion proves saveFile persisted the in-memory edit. The
+    # screenshot-only check can misread the small tab dirty-dot transition.
+    skipLlmVerify: true