fix(tests): retry stopDetector to absorb AD plugin start/stop race

The live Anomaly Detection round-trip test (create -> get -> preview ->
start -> stop) intermittently failed on OpenSearch 2.19.5 (run
28338345697) with "Fail to stop detector". The AD plugin registers the
real-time job asynchronously in .opendistro-anomaly-detector-jobs after
startDetector acknowledges, so an immediate stop can transiently find no
running job.

- Wrap stopDetector in a bounded retry helper (10 x 1s) that tolerates
  only the specific "Fail to stop detector" error, mirroring the
  existing `waitFor` idiom (CCRSpec/ReindexSpec/TasksSpec). Genuine
  errors still surface.
- Apply the helper to the OS1/OS2/OS3 live round-trip tests.

Author: blackheaven

tests/Test/AnomalyDetectionSpec.hs

@@ -3,28 +3,29 @@
 
 module Test.AnomalyDetectionSpec (spec) where
 
+import Control.Concurrent (threadDelay)
 import Control.Monad (void)
 import Control.Monad.Catch (bracket_)
 import Control.Monad.IO.Class (liftIO)
 import Data.Aeson (Value (..), decode, encode, object, (.=))
-import qualified Data.ByteString.Lazy.Char8 as LBS
+import Data.ByteString.Lazy.Char8 qualified as LBS
 import Data.Fixed (Pico)
-import qualified Data.List as L
+import Data.List qualified as L
 import Data.Maybe (isJust)
-import qualified Data.Text as T
+import Data.Text qualified as T
 import Data.Time.Calendar (toGregorian)
 import Data.Time.Clock (UTCTime (..))
 import Data.Time.Clock.POSIX (getPOSIXTime, posixSecondsToUTCTime)
 import Data.Time.LocalTime (TimeOfDay (..), timeToTimeOfDay)
-import qualified Database.Bloodhound.OpenSearch1.Client as OS1Client
-import qualified Database.Bloodhound.OpenSearch1.Requests as OS1Requests
-import qualified Database.Bloodhound.OpenSearch1.Types as OS1Types
-import qualified Database.Bloodhound.OpenSearch2.Client as OS2Client
-import qualified Database.Bloodhound.OpenSearch2.Requests as OS2Requests
-import qualified Database.Bloodhound.OpenSearch2.Types as OS2Types
-import qualified Database.Bloodhound.OpenSearch3.Client as OS3Client
-import qualified Database.Bloodhound.OpenSearch3.Requests as OS3Requests
-import qualified Database.Bloodhound.OpenSearch3.Types as OS3Types
+import Database.Bloodhound.OpenSearch1.Client qualified as OS1Client
+import Database.Bloodhound.OpenSearch1.Requests qualified as OS1Requests
+import Database.Bloodhound.OpenSearch1.Types qualified as OS1Types
+import Database.Bloodhound.OpenSearch2.Client qualified as OS2Client
+import Database.Bloodhound.OpenSearch2.Requests qualified as OS2Requests
+import Database.Bloodhound.OpenSearch2.Types qualified as OS2Types
+import Database.Bloodhound.OpenSearch3.Client qualified as OS3Client
+import Database.Bloodhound.OpenSearch3.Requests qualified as OS3Requests
+import Database.Bloodhound.OpenSearch3.Types qualified as OS3Types
 import TestsUtils.Common
   ( os1OnlyIT,
     os2OnlyIT,
@@ -1074,7 +1075,7 @@ spec = do
                       expectationFailure
                         ("startDetector failed: " <> T.unpack (errorMessage e))
                     Right _ -> pure ()
-                stopResp <- OS1Client.stopDetector detectorId False
+                stopResp <- stopDetectorWithRetry (OS1Client.stopDetector detectorId False)
                 liftIO $
                   case stopResp of
                     Left e ->
@@ -1086,7 +1087,7 @@ spec = do
                 liftIO $ expectationFailureOnLeft updateResp "updateDetector failed:"
                 startResp <- OS1Client.startDetector detectorId Nothing
                 liftIO $ expectationFailureOnLeft startResp "startDetector failed:"
-                stopResp <- OS1Client.stopDetector detectorId False
+                stopResp <- stopDetectorWithRetry (OS1Client.stopDetector detectorId False)
                 liftIO $ expectationFailureOnLeft stopResp "stopDetector failed:"
                 delResp <- OS1Client.deleteDetector detectorId
                 liftIO $ expectationFailureOnLeft delResp "deleteDetector failed:"
@@ -1161,7 +1162,7 @@ spec = do
                       expectationFailure
                         ("startDetector failed: " <> T.unpack (errorMessage e))
                     Right _ -> pure ()
-                stopResp <- OS2Client.stopDetector detectorId False
+                stopResp <- stopDetectorWithRetry (OS2Client.stopDetector detectorId False)
                 liftIO $
                   case stopResp of
                     Left e ->
@@ -1173,7 +1174,7 @@ spec = do
                 liftIO $ expectationFailureOnLeft updateResp "updateDetector failed:"
                 startResp <- OS2Client.startDetector detectorId Nothing
                 liftIO $ expectationFailureOnLeft startResp "startDetector failed:"
-                stopResp <- OS2Client.stopDetector detectorId False
+                stopResp <- stopDetectorWithRetry (OS2Client.stopDetector detectorId False)
                 liftIO $ expectationFailureOnLeft stopResp "stopDetector failed:"
                 delResp <- OS2Client.deleteDetector detectorId
                 liftIO $ expectationFailureOnLeft delResp "deleteDetector failed:"
@@ -1248,7 +1249,7 @@ spec = do
                       expectationFailure
                         ("startDetector failed: " <> T.unpack (errorMessage e))
                     Right _ -> pure ()
-                stopResp <- OS3Client.stopDetector detectorId False
+                stopResp <- stopDetectorWithRetry (OS3Client.stopDetector detectorId False)
                 liftIO $
                   case stopResp of
                     Left e ->
@@ -1260,7 +1261,7 @@ spec = do
                 liftIO $ expectationFailureOnLeft updateResp "updateDetector failed:"
                 startResp <- OS3Client.startDetector detectorId Nothing
                 liftIO $ expectationFailureOnLeft startResp "startDetector failed:"
-                stopResp <- OS3Client.stopDetector detectorId False
+                stopResp <- stopDetectorWithRetry (OS3Client.stopDetector detectorId False)
                 liftIO $ expectationFailureOnLeft stopResp "stopDetector failed:"
                 delResp <- OS3Client.deleteDetector detectorId
                 liftIO $ expectationFailureOnLeft delResp "deleteDetector failed:"
@@ -1326,3 +1327,24 @@ expectationFailureOnLeft r prefix =
   case r of
     Left e -> expectationFailure (prefix <> " " <> T.unpack (errorMessage e))
     Right _ -> pure ()
+
+-- | Tolerate the Anomaly Detection plugin's start\/stop race:
+-- 'startDetector' returns as soon as it acknowledges the start, but the
+-- real-time job is registered asynchronously in the
+-- @.opendistro-anomaly-detector-jobs@ index, so an immediate
+-- 'stopDetector' can transiently receive "Fail to stop detector" from the
+-- server. Retry only that specific error, bounded so a genuine failure
+-- still surfaces. Mirrors the 'waitFor' idiom in "Test.CCRSpec".
+stopDetectorWithRetry ::
+  (MonadIO m) =>
+  m (Either EsError a) ->
+  m (Either EsError a)
+stopDetectorWithRetry stop = go (0 :: Int)
+  where
+    go n = do
+      r <- stop
+      case r of
+        Left e
+          | "Fail to stop detector" `T.isInfixOf` errorMessage e && n < 10 ->
+              liftIO (threadDelay 1000000) >> go (n + 1)
+        _ -> pure r