Skip to content

Commit

Permalink
OSM
Browse files Browse the repository at this point in the history
  • Loading branch information
docuracy committed Jan 25, 2025
1 parent f48552d commit 77eb358
Showing 1 changed file with 3 additions and 11 deletions.
14 changes: 3 additions & 11 deletions vespa/repository/api/ingestion/streamer.py
Original file line number Diff line number Diff line change
Expand Up @@ -185,11 +185,11 @@ async def async_generator():

return async_generator()

async def _parse_xml_stream(self, stream):
def _parse_xml_stream(self, stream):
"""
Asynchronous parser for XML streams.
"""
def parse():
async def async_generator():
# Use xmltodict's streaming mode to process XML elements one by one
try:
for doc in xmltodict.parse(stream, items=("item", self.item_path), stream=True):
Expand All @@ -198,16 +198,8 @@ def parse():
self.logger.error(f"Failed to parse XML stream. Error: {e}")
raise

# Run parse() in a separate thread using asyncio.to_thread
parse_gen = await asyncio.to_thread(parse)

# Wrap the generator into an async iterator
async def async_gen():
for item in parse_gen:
yield item

# Return the async iterator
return async_gen()
return async_generator()

def _split_triple(self, line):
parts = line.rstrip(' .').split(' ', 2)
Expand Down

0 comments on commit 77eb358

Please sign in to comment.