Fix flaky :plugins:repository-azure:internalClusterTest by waiting for Azure fixture readiness and cleaning stale state

plugins/repository-azure/src/internalClusterTest/java/org/opensearch/repositories/azure/AzureStorageCleanupThirdPartyTests.java

@@ -37,6 +37,7 @@
 import com.azure.storage.blob.BlobServiceClient;
 import com.azure.storage.blob.models.BlobStorageException;
 import org.opensearch.action.ActionRunnable;
+import org.opensearch.action.support.IndicesOptions;
 import org.opensearch.action.support.PlainActionFuture;
 import org.opensearch.common.collect.Tuple;
 import org.opensearch.common.settings.MockSecureSettings;
@@ -49,9 +50,11 @@
 import org.opensearch.secure_sm.AccessController;
 import org.opensearch.test.OpenSearchIntegTestCase;
 import org.junit.AfterClass;
+import org.junit.BeforeClass;
 
 import java.net.HttpURLConnection;
 import java.util.Collection;
+import java.util.concurrent.TimeUnit;
 import java.util.function.Supplier;
 
 import reactor.core.scheduler.Schedulers;
@@ -60,11 +63,109 @@
 import static org.hamcrest.Matchers.not;
 
 public class AzureStorageCleanupThirdPartyTests extends AbstractThirdPartyRepositoryTestCase {
+
+    @AfterClass
+    public static void cleanupAzureBlobs() throws Exception {
+        try {
+            String container = System.getProperty("test.azure.container");
+            String account = System.getProperty("test.azure.account");
+
+            if (Strings.isNullOrEmpty(container) || Strings.isNullOrEmpty(account)) {
+                return;
+            }
+
+            MockSecureSettings secureSettings = new MockSecureSettings();
+            secureSettings.setString("azure.client.default.account", account);
+
+            if (Strings.hasText(System.getProperty("test.azure.sas_token"))) {
+                secureSettings.setString("azure.client.default.sas_token", System.getProperty("test.azure.sas_token"));
+            } else {
+                secureSettings.setString("azure.client.default.key", System.getProperty("test.azure.key", ""));
+            }
+
+            Settings settings = Settings.builder().setSecureSettings(secureSettings).build();
+
+            try (AzureStorageService storageService = new AzureStorageService(settings)) {
+                Tuple<BlobServiceClient, Supplier<Context>> client = storageService.client("default");
+                BlobContainerClient blobContainer = client.v1().getBlobContainerClient(container);
+
+                if (!blobContainer.exists()) {
+                    return;
+                }
+
+                blobContainer.listBlobs().forEach(b -> blobContainer.getBlobClient(b.getName()).delete());
+
+                assertBusy(() -> assertFalse(blobContainer.listBlobs().iterator().hasNext()), 30, TimeUnit.SECONDS);
+            }
+        } catch (Exception ignored) {
+            // CI teardown
+        }
+    }
+
     @AfterClass
     public static void shutdownSchedulers() {
         Schedulers.shutdownNow();
     }
 
+    @BeforeClass
+    public static void waitForAzureFixtureReady() throws Exception {
+        assertBusy(() -> {
+            assertThat(System.getProperty("test.azure.account"), not(blankOrNullString()));
+
+            final boolean hasSasToken = Strings.hasText(System.getProperty("test.azure.sas_token"));
+            if (hasSasToken == false) {
+                assertThat(System.getProperty("test.azure.key"), not(blankOrNullString()));
+            } else {
+                assertThat(System.getProperty("test.azure.key"), blankOrNullString());
+            }
+
+            assertThat(System.getProperty("test.azure.container"), not(blankOrNullString()));
+            assertThat(System.getProperty("test.azure.base"), not(blankOrNullString()));
+        }, 60, TimeUnit.SECONDS);
+
+        assertBusy(() -> {
+            String container = System.getProperty("test.azure.container");
+
+            MockSecureSettings secureSettings = new MockSecureSettings();
+            secureSettings.setString("azure.client.default.account", System.getProperty("test.azure.account"));
+
+            final boolean hasSasToken = Strings.hasText(System.getProperty("test.azure.sas_token"));
+            if (hasSasToken) {
+                secureSettings.setString("azure.client.default.sas_token", System.getProperty("test.azure.sas_token", ""));
+            } else {
+                secureSettings.setString("azure.client.default.key", System.getProperty("test.azure.key", ""));
+            }
+
+            Settings s = Settings.builder().setSecureSettings(secureSettings).build();
+
+            try (AzureStorageService storageService = new AzureStorageService(s)) {
+                Tuple<BlobServiceClient, Supplier<Context>> client = storageService.client("default");
+                BlobContainerClient blobContainer = client.v1().getBlobContainerClient(container);
+
+                blobContainer.existsWithResponse(null, client.v2().get());
+            } catch (Exception e) {
+                throw new AssertionError("Azure container not ready yet: " + e.getMessage(), e);
+            }
+        }, 60, TimeUnit.SECONDS);
+    }
+
+    @Override
+    public void setUp() throws Exception {
+        super.setUp();
+        ensureGreen("_all");
+    }
+
+    @Override
+    public void tearDown() throws Exception {
+        try {
+            client().admin().indices().prepareDelete("*").setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN).get();
+
+            client().admin().cluster().prepareDeleteRepository("_all").get();
+        } finally {
+            super.tearDown();
+        }
+    }
+
     @Override
     protected Collection<Class<? extends Plugin>> getPlugins() {
         return pluginList(AzureRepositoryPlugin.class);