Skip to content

Commit

Permalink
Move configurable retry_on_conflict setting from app.cfg.example to s…
Browse files Browse the repository at this point in the history
…earch-config.yaml.example.
  • Loading branch information
kburke committed Jul 18, 2024
1 parent ca09519 commit d62e451
Show file tree
Hide file tree
Showing 4 changed files with 5 additions and 8 deletions.
7 changes: 3 additions & 4 deletions src/hubmap_translator.py
Original file line number Diff line number Diff line change
Expand Up @@ -66,8 +66,7 @@ class Translator(TranslatorInterface):
failed_entity_api_calls = []
failed_entity_ids = []

def __init__(self, indices, app_client_id, app_client_secret, token, ontology_api_base_url:str=None
, es_retry_on_conflict_freq:int=0):
def __init__(self, indices, app_client_id, app_client_secret, token, ontology_api_base_url:str=None):
try:
self.ingest_api_soft_assay_url = indices['ingest_api_soft_assay_url'].strip('/')
self.indices: dict = {}
Expand All @@ -82,7 +81,7 @@ def __init__(self, indices, app_client_id, app_client_secret, token, ontology_ap
self.INDICES: dict = {'default_index': self.DEFAULT_INDEX_WITHOUT_PREFIX, 'indices': self.indices}
self.DEFAULT_ENTITY_API_URL = self.INDICES['indices'][self.DEFAULT_INDEX_WITHOUT_PREFIX]['document_source_endpoint'].strip('/')
self._ontology_api_base_url = ontology_api_base_url
self.es_retry_on_conflict_freq = es_retry_on_conflict_freq
self.es_retry_on_conflict_param_value = indices['es_retry_on_conflict_param_value']

self.indexer = Indexer(self.indices, self.DEFAULT_INDEX_WITHOUT_PREFIX)

Expand Down Expand Up @@ -376,7 +375,7 @@ def _directly_modify_related_entities( self, es_url:str, es_index:str, entity_i
f' \"lang\": \"painless\",' \
f' \"source\": \"{painless_query}\",' \
f' \"params\": {{' \
f' \"retry_on_conflict\": {self.es_retry_on_conflict_freq},' \
f' \"retry_on_conflict\": {self.es_retry_on_conflict_param_value},' \
f' \"modified_entity_uuid\": \"<TARGET_MODIFIED_ENTITY_UUID>\",' \
f' \"revised_related_entity\": <THIS_REVISED_ENTITY>' \
f' }}' \
Expand Down
3 changes: 0 additions & 3 deletions src/instance/app.cfg.example
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,6 @@ AWS_OBJECT_URL_EXPIRATION_IN_SECS = 60*60 # 1 hour
# returned in the response to avoid the AWS Gateway 10Mb constraint
LARGE_RESPONSE_THRESHOLD = 9*(2**20) + 900*(2**10) #9.9Mb

# ElasticSearch retry_on_conflict value. Positive integer or zero.
ES_RETRY_ON_CONFLICT_FREQ = 5

# Base URL for the Ontology API
ONTOLOGY_API_BASE_URL = 'https://ontology-api.dev.hubmapconsortium.org'

Expand Down
2 changes: 2 additions & 0 deletions src/instance/search-config.yaml.example
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@
ingest_api_soft_assay_url: https://ingest-api.dev.hubmapconsortium.org/assaytype/
# default index name for endpoints that don't specify an index
default_index: entities
# ElasticSearch retry_on_conflict value. Positive integer or zero.
es_retry_on_conflict_param_value: 5

# specify multiple indices
indices:
Expand Down
1 change: 0 additions & 1 deletion src/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,6 @@
config['AWS_S3_BUCKET_NAME'] = app.config['AWS_S3_BUCKET_NAME']
config['AWS_S3_OBJECT_PREFIX'] = app.config['AWS_S3_OBJECT_PREFIX']
config['AWS_OBJECT_URL_EXPIRATION_IN_SECS'] = app.config['AWS_OBJECT_URL_EXPIRATION_IN_SECS']
config['ES_RETRY_ON_CONFLICT_FREQ'] = app.config['ES_RETRY_ON_CONFLICT_FREQ']
config['LARGE_RESPONSE_THRESHOLD'] = app.config['LARGE_RESPONSE_THRESHOLD']
config['PARAM_SEARCH_RECOGNIZED_ENTITIES_BY_INDEX'] = app.config['PARAM_SEARCH_RECOGNIZED_ENTITIES_BY_INDEX']
config['ONTOLOGY_API_BASE_URL'] = app.config['ONTOLOGY_API_BASE_URL'].strip('/')
Expand Down

0 comments on commit d62e451

Please sign in to comment.