From f29d2876a126e5c9cf41ee571d0ec52a4c2fd58e Mon Sep 17 00:00:00 2001
From: sax3l <simon@siax.io>
Date: Mon, 2 Nov 2026 20:29:30 +0100
Subject: [PATCH] ci(integration): clear the OpenSearch read-only block, not
 just the watermark

Disabling cluster.routing.allocation.disk.threshold_enabled does not clear a
cluster already in flood stage: it keeps cluster.blocks.read_only_allow_delete
and every write returns 403 index_create_block_exception (run 389046, still red
after the previous fix). Clear that block explicitly on every attempt, set the
threshold off at boot, and print the probe/settings bodies for future triage.

Verified locally against opensearch:2.11.1: read-only block -> 403, then
PUT {... read_only_allow_delete: null} -> index creation 200.
---
 .gitea/workflows/ci.yml | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml
index bc3a767..6b6ef5e 100644
--- a/.gitea/workflows/ci.yml
+++ b/.gitea/workflows/ci.yml
@@ -97,6 +97,9 @@ jobs:
           # cluster_name (the OpenSearch image maps dotted env keys to
           # settings).
           cluster.name: opt_ci_${{ github.run_id }}_${{ github.run_attempt }}
+          # Throwaway CI cluster: never let the runner's disk fill turn indices
+          # read-only (the 403 index_create_block_exception seen in run 388536).
+          cluster.routing.allocation.disk.threshold_enabled: "false"
         # `--workdir` is REQUIRED (measured 2026-08-31, PR #25 CI run 7745,
         # job "integration"): this runner's act_runner (v0.6.1) merges the
         # JOB's WorkingDir (`/workspace/sax3l/0pt`) into every SERVICE
@@ -161,15 +164,22 @@ jobs:
           # is throwaway CI state, so disable the disk threshold and PROVE that
           # index creation works before the blocking tests run.
           for attempt in 1 2 3 4 5; do
-            curl -fsS --max-time 10 -X PUT "http://$FOUND:9200/_cluster/settings" \
+            # Disabling the watermark alone does NOT clear a cluster that is
+            # already read-only: the flood stage sets
+            # cluster.blocks.read_only_allow_delete and every write keeps
+            # returning 403. Clear that block explicitly every attempt.
+            curl -sS --max-time 10 -X PUT "http://$FOUND:9200/_cluster/settings" \
               -H 'Content-Type: application/json' \
-              -d '{"persistent":{"cluster.routing.allocation.disk.threshold_enabled":false}}' >/dev/null || true
-            if curl -fsS --max-time 10 -X PUT "http://$FOUND:9200/opt-ci-probe-${{ github.run_id }}" >/dev/null; then
+              -d '{"persistent":{"cluster.routing.allocation.disk.threshold_enabled":false,"cluster.blocks.read_only_allow_delete":null}}' >/tmp/os-settings.json || true
+            cat /tmp/os-settings.json 2>/dev/null || true
+            probe_status=$(curl -sS --max-time 10 -o /tmp/os-probe.json -w '%{http_code}' -X PUT "http://$FOUND:9200/opt-ci-probe-${{ github.run_id }}")
+            if [ "$probe_status" = "200" ]; then
               curl -fsS --max-time 10 -X DELETE "http://$FOUND:9200/opt-ci-probe-${{ github.run_id }}" >/dev/null || true
               echo "OpenSearch accepts index creation"
               break
             fi
-            echo "index creation not ready (attempt $attempt/5) — waiting"
+            echo "index creation not ready (attempt $attempt/5, HTTP $probe_status) — waiting"
+            cat /tmp/os-probe.json 2>/dev/null || true
             [ "$attempt" = "5" ] && { echo "FATAL: OpenSearch still refuses index creation"; exit 1; }
             sleep 10
           done
-- 
2.54.0