[r p] Add server-side retries for scripted ES document updates (#6648)

This change alleviates the contention (#6648) a little bit but doesn't resolve it. Once #6648 is fully addressed, this should significantly reduce the remaining bits of contention, of which there were just a few dozen incidents per reindex before we fixed the absence of donor and protocol replicas, which are the main drivers of the contention.

src/azul/indexer/document.py

@@ -1330,6 +1330,8 @@ def to_index(self,
             result['body'] = self._body(field_types[coordinates.entity.catalog])
         if self.version is not None:
             result['if_seq_no'], result['if_primary_term'] = self.version
+        if self.op_type is OpType.update:
+            result['params'] = {'retry_on_conflict': 3}
         return result
 
     def _body(self, field_types: FieldTypes) -> JSON:

src/azul/indexer/index_service.py

@@ -890,6 +890,7 @@ def expand_action(doc: Any) -> tuple[dict[str, Any], dict[str, Any] | None]:
             # optional document source.
             assert isinstance(doc, Document), doc
             action = dict(doc.to_index(self.catalog, self.field_types))
+            action.update(action.pop('params', {}))
             action['_index'] = action.pop('index')
             action['_id'] = action.pop('id')
             body = action.pop('body', None)