Fix Flaky results on Library and Gateway controller test

tests/e2e/gatewaycontroller/gateway_controller_test.go

@@ -35,6 +35,7 @@ import (
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	"istio.io/istio/pkg/ptr"
@@ -215,10 +216,20 @@ spec:
 		})
 
 		It("curl client pod is ready", func(ctx SpecContext) {
-			Eventually(func() error {
-				return common.CheckPodsReady(ctx, cl, gatewayNamespace)
+			// Check only the curl-client pod
+			Eventually(func(g Gomega) {
+				pod := &corev1.Pod{}
+				g.Expect(cl.Get(ctx, kube.Key("curl-client", gatewayNamespace), pod)).To(Succeed())
+				var ready bool
+				for _, cond := range pod.Status.Conditions {
+					if cond.Type == corev1.PodReady && cond.Status == corev1.ConditionTrue {
+						ready = true
+						break
+					}
+				}
+				g.Expect(ready).To(BeTrue(), "curl-client pod is not ready")
 			}).Should(Succeed())
-			Success("All pods in gateway namespace are ready")
+			Success("Curl client pod is ready")
 		})
 	})
 

tests/e2e/library/library_reconcile_test.go

@@ -275,12 +275,18 @@ var _ = Describe("Library Reconciliation", Label("library", "reconciliation"), O
 			webhookKey := types.NamespacedName{
 				Name: fmt.Sprintf("istio-validator-%s-%s", revision, namespace),
 			}
-			webhook := &admissionv1.ValidatingWebhookConfiguration{}
-			Expect(cl.Get(ctx, webhookKey, webhook)).To(Succeed())
 
-			webhook.Webhooks[0].Name = "xyz.xyz.xyz"
-			webhook.Webhooks[0].FailurePolicy = ptr.Of(admissionv1.Fail)
-			Expect(cl.Update(ctx, webhook)).To(Succeed())
+			// Refresh and retry: the background reconciler may update the webhook
+			// concurrently (causing a 409 conflict on Update) or transiently clear
+			// the Webhooks list, so we assert rather than index directly.
+			Eventually(func(g Gomega) {
+				webhook := &admissionv1.ValidatingWebhookConfiguration{}
+				g.Expect(cl.Get(ctx, webhookKey, webhook)).To(Succeed())
+				g.Expect(webhook.Webhooks).NotTo(BeEmpty(), "webhook has no entries")
+				webhook.Webhooks[0].Name = "xyz.xyz.xyz"
+				webhook.Webhooks[0].FailurePolicy = ptr.Of(admissionv1.Fail)
+				g.Expect(cl.Update(ctx, webhook)).To(Succeed())
+			}).Should(Succeed())
 
 			Eventually(func(g Gomega) {
 				restored := &admissionv1.ValidatingWebhookConfiguration{}

tests/e2e/util/common/e2e_utils.go

@@ -565,11 +565,23 @@ func withClusterName(m string, k kubectl.Kubectl) string {
 	return m + " on " + k.ClusterName
 }
 
-func CheckPodConnectivity(podName, containerName, srcNamespace, destNamespace string, k kubectl.Kubectl) {
+// CheckPodConnectivityWithError tests connectivity from podName to httpbin in destNamespace
+// and returns an error instead of calling Expect directly. This allows callers wrapped in
+// Eventually to retry on transient failures (e.g. 503 during proxy startup/upgrade).
+func CheckPodConnectivityWithError(podName, containerName, srcNamespace, destNamespace string, k kubectl.Kubectl) error {
 	command := fmt.Sprintf(`curl -o /dev/null -s -w "%%{http_code}\n" httpbin.%s.svc.cluster.local:8000/get`, destNamespace)
 	response, err := k.WithNamespace(srcNamespace).Exec(podName, containerName, command)
-	Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("error connecting to the %q pod", podName))
-	Expect(response).To(ContainSubstring("200"), fmt.Sprintf("Unexpected response from %s pod", podName))
+	if err != nil {
+		return fmt.Errorf("error connecting to the %q pod: %w", podName, err)
+	}
+	if !strings.Contains(response, "200") {
+		return fmt.Errorf("unexpected response from %s pod: %s", podName, strings.TrimSpace(response))
+	}
+	return nil
+}
+
+func CheckPodConnectivity(podName, containerName, srcNamespace, destNamespace string, k kubectl.Kubectl) {
+	Expect(CheckPodConnectivityWithError(podName, containerName, srcNamespace, destNamespace, k)).To(Succeed())
 }
 
 func HaveContainersThat(matcher types.GomegaMatcher) types.GomegaMatcher {

tests/e2e/util/common/workload_validator.go

@@ -120,9 +120,11 @@ func (w *WorkloadValidator) ValidateConnectivity(ctx context.Context) error {
 		return fmt.Errorf("no pods found in %s namespace", w.Namespace)
 	}
 
-	// Test connectivity from sleep to httpbin
-	CheckPodConnectivity(sleepPods.Items[0].Name, SleepContainerName, w.Namespace, HttpbinNamespace, w.K)
-	return nil
+	// Test connectivity from sleep to httpbin. Use the error-returning form so that
+	// transient failures (e.g. 503 during proxy startup/upgrade) are propagated as
+	// errors and can be retried by the caller's Eventually block rather than
+	// immediately failing the test via a bare Expect.
+	return CheckPodConnectivityWithError(sleepPods.Items[0].Name, SleepContainerName, w.Namespace, HttpbinNamespace, w.K)
 }
 
 // ValidateProxyVersion validates proxy version based on dataplane mode