Added retry. (#453)

* Added retry.

Signed-off-by: dblock <[email protected]>

.cspell

@@ -1,52 +1,8 @@
-APIV
-Boxplot
-Bobg
-Boundsdouble
-CBOR
-DNFOF
-Decompounder
-Downsample
-Downsampling
-Ewma
-Fielddata
-Formattable
-Fragmenter
-Geohex
-Gsub
-Intertransport
-Jelinek
-Kuromoji
-Lovins
-Lucene
-Millis
-Moneyball
-Multisearch
-Moneyball
-Nanos
-Nori
-ONNX
-OPENSEARCH
-OSCPU
-Oversample
-Rebalance
-Reindex
-Rethrottle
-Rolespan
-Rollup
-Rollups
-Slowlog
-Sorani
-Stringifiedboolean
-Stringifiedinteger
-Tfidf
-Tika
-Translog
-Unmanaged
-Yrtsd
 aarch
 actiongroup
 actiongroups
 aggregatable
+APIV
 argjson
 asciifolding
 authc
@@ -56,63 +12,92 @@ autocut
 backendroles
 backpressure
 beider
+Bobg
+Boundsdouble
 boxplot
+Boxplot
 caverphone
+CBOR
 charfilters
 chisquared
 chuhlomin
 cpuacct
+cuserr
 daitch
 dangoslen
 dashboardsinfo
+datarows
 decompounder
+Decompounder
 dedup
 determinized
+distilbert
+DNFOF
 docvalue
+Downsample
 downsampling
+Downsampling
 evals
 ewma
+Ewma
 faiss
 fielddata
+Fielddata
 forcemerge
+Formattable
 fragmenter
+Fragmenter
 freqs
 generateonbehalfoftoken
 geohash
 geohex
+Geohex
 geoip
 geotile
 gsub
+Gsub
 haasephonetik
 heteroscedastic
 homoscedastic
 hotthreads
+huggingface
 hybridfs
 integ
 internalusers
+Intertransport
+Jelinek
 kibanainfo
 kibanaserver
 koelnerphonetik
 kstem
 kuromoji
+Kuromoji
 languageset
+Lovins
 lucene
+Lucene
 lycheeverse
 marvinpinto
 metaphone
 mget
 millis
+Millis
 mlockall
 mmap
 mmapfs
 mmdb
 mokotoff
+Moneyball
+Moneyball
 msearch
+msmarco
 mtermvectors
 mult
+Multisearch
 multitenancy
 mxyz
 nanos
+Nanos
 nbest
 nfkc
 ngram
@@ -122,13 +107,18 @@ nodeattrs
 nodesdn
 noops
 nori
+Nori
 nysiis
+ONNX
 opendistro
 opensearch
+OPENSEARCH
 opensearchproject
 opensearchstaging
 ords
+OSCPU
 oversample
+Oversample
 performanceanalyzer
 permissionsinfo
 pipefail
@@ -137,38 +127,54 @@ prirep
 rawfile
 readingform
 rebalance
+Rebalance
 recoverysource
 reindex
+Reindex
 relo
 reloadcerts
 remotestore
 rethrottle
+Rethrottle
 rolesmapping
+Rolespan
 rollup
+Rollup
 rollups
+Rollups
 romaji
 roundtrips
+Rudnick
 ruleset
 scriptless
 securityconfig
 slowlog
+Slowlog
 slurpfile
 snapshotted
 softmax
+Sorani
 sslinfo
 stoptags
 stopwords
+Stringifiedboolean
+Stringifiedinteger
 subqueries
 subschemas
 subword
+syserr
 tdigest
 tenantinfo
 termvectors
 tfidf
+Tfidf
+Tika
 tokenfilters
 translog
+Translog
 tubone
 unigrams
+Unmanaged
 unmatch
 untriaged
 updateable
@@ -177,7 +183,4 @@ urldecode
 vectory
 whoamiprotected
 wordnet
-datarows
-syserr
-cuserr
-Rudnick
+Yrtsd

CHANGELOG.md

@@ -55,6 +55,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Added `plugins` to NodeInfoSettings ([#442](https://github.com/opensearch-project/opensearch-api-specification/pull/442)) 
 - Added test coverage ([#443](https://github.com/opensearch-project/opensearch-api-specification/pull/443))
 - Added `--opensearch-version` to `merger` that excludes schema elements per semver ([#428](https://github.com/opensearch-project/opensearch-api-specification/pull/428))
+- Added `retry` to `tester` to support asynchronous tasks ([453](https://github.com/opensearch-project/opensearch-api-specification/pull/453))
 
 ### Changed
 

TESTING_GUIDE.md

@@ -5,6 +5,7 @@
     - [Simple Test Story](#simple-test-story)
     - [Using Output from Previous Chapters](#using-output-from-previous-chapters)
     - [Managing Versions](#managing-versions)
+    - [Waiting for Tasks](#waiting-for-tasks)
 <!-- TOC -->
 
 # Spec Testing Guide
@@ -153,3 +154,24 @@ It's common to add a feature to the next version of OpenSearch. When adding a ne
 ```
 
 The [integration test workflow](.github/workflows/test-spec.yml) runs a matrix of OpenSearch versions, including the next version. Please check whether the workflow needs an update when adding version-specific tests.
+
+### Waiting for Tasks
+
+Some APIs behave asynchronously and may require a test to wait for a task to complete. This can be achived with a combination of `payload` and `retry`. 
+
+For example, an ML task returns `CREATED` when created, and `COMPLETED` when it's done. The example below will retry 3 times with an interval of 30 seconds until the task is complete. The default wait time is 1s.
+
+```yaml
+  - synopsis: Wait for task.
+    path: /_plugins/_ml/tasks/{task_id}
+    method: GET
+    parameters:
+      task_id: ${create_model.task_id}
+    response:
+      status: 200
+      payload:
+        state: COMPLETED
+    retry:
+      count: 3
+      wait: 30000
+```

json_schemas/test_story.schema.yaml

@@ -81,6 +81,8 @@ definitions:
         $ref: '#/definitions/Output'
       version:
         $ref: '#/definitions/Version'
+      retry:
+        $ref: '#/definitions/Retry'
     required: [method, path]
 
   Output:
@@ -100,6 +102,21 @@ definitions:
       The semver range to execute the story or chapter against.
     type: string
 
+  Retry:
+    description: |
+      Number of times to retry on error.
+    oneOf:
+      - type: object
+        properties:
+          count: 
+            type: integer
+            description: Number of retries.
+          wait:
+            type: integer
+            description: Number of milliseconds to wait before retrying.
+        required:
+          - count
+
   RequestBody:
     type: object
     properties: