diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index bc3a767..6b6ef5e 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -97,6 +97,9 @@ jobs: # cluster_name (the OpenSearch image maps dotted env keys to # settings). cluster.name: opt_ci_${{ github.run_id }}_${{ github.run_attempt }} + # Throwaway CI cluster: never let the runner's disk fill turn indices + # read-only (the 403 index_create_block_exception seen in run 388536). + cluster.routing.allocation.disk.threshold_enabled: "false" # `--workdir` is REQUIRED (measured 2026-08-31, PR #25 CI run 7745, # job "integration"): this runner's act_runner (v0.6.1) merges the # JOB's WorkingDir (`/workspace/sax3l/0pt`) into every SERVICE @@ -161,15 +164,22 @@ jobs: # is throwaway CI state, so disable the disk threshold and PROVE that # index creation works before the blocking tests run. for attempt in 1 2 3 4 5; do - curl -fsS --max-time 10 -X PUT "http://$FOUND:9200/_cluster/settings" \ + # Disabling the watermark alone does NOT clear a cluster that is + # already read-only: the flood stage sets + # cluster.blocks.read_only_allow_delete and every write keeps + # returning 403. Clear that block explicitly every attempt. + curl -sS --max-time 10 -X PUT "http://$FOUND:9200/_cluster/settings" \ -H 'Content-Type: application/json' \ - -d '{"persistent":{"cluster.routing.allocation.disk.threshold_enabled":false}}' >/dev/null || true - if curl -fsS --max-time 10 -X PUT "http://$FOUND:9200/opt-ci-probe-${{ github.run_id }}" >/dev/null; then + -d '{"persistent":{"cluster.routing.allocation.disk.threshold_enabled":false,"cluster.blocks.read_only_allow_delete":null}}' >/tmp/os-settings.json || true + cat /tmp/os-settings.json 2>/dev/null || true + probe_status=$(curl -sS --max-time 10 -o /tmp/os-probe.json -w '%{http_code}' -X PUT "http://$FOUND:9200/opt-ci-probe-${{ github.run_id }}") + if [ "$probe_status" = "200" ]; then curl -fsS --max-time 10 -X DELETE "http://$FOUND:9200/opt-ci-probe-${{ github.run_id }}" >/dev/null || true echo "OpenSearch accepts index creation" break fi - echo "index creation not ready (attempt $attempt/5) — waiting" + echo "index creation not ready (attempt $attempt/5, HTTP $probe_status) — waiting" + cat /tmp/os-probe.json 2>/dev/null || true [ "$attempt" = "5" ] && { echo "FATAL: OpenSearch still refuses index creation"; exit 1; } sleep 10 done