Skip to content

Commit

Permalink
Just add approve_timeout to the python script
Browse files Browse the repository at this point in the history
  • Loading branch information
anna-parker committed Aug 21, 2024
1 parent dca0767 commit 8b932a5
Show file tree
Hide file tree
Showing 2 changed files with 22 additions and 5 deletions.
9 changes: 5 additions & 4 deletions ingest/Snakefile
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ COLUMN_MAPPING = config["column_mapping"]
LOG_LEVEL = config.get("log_level", "INFO")
NCBI_API_KEY = os.getenv("NCBI_API_KEY")
FILTER_FASTA_HEADERS = config.get("filter_fasta_headers", None)
APPROVE_TIMEOUT = config.get("approve_timeout", '1m')
APPROVE_TIMEOUT = config.get("approve_timeout", '1') #time in minutes


def rename_columns(input_file, output_file, mapping=COLUMN_MAPPING):
Expand Down Expand Up @@ -448,9 +448,10 @@ rule approve:
approve_timeout=APPROVE_TIMEOUT,
shell:
"""
timeout --preserve-status -s SIGTERM {params.approve_timeout} \
bash -c "python {input.script} \
python {input.script} \
--mode approve \
--config-file {input.config} \
--log-level {params.log_level}" \
--log-level {params.log_level} \
--approve-timeout {params.approve_timeout}
touch results/approved
"""
18 changes: 17 additions & 1 deletion ingest/scripts/call_loculus.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,15 @@
import os
from collections import defaultdict
from dataclasses import dataclass
from datetime import datetime, timedelta
from http import HTTPMethod
from pathlib import Path
from time import sleep
from typing import Any, Literal

import click
import jsonlines
import pytz
import requests
import yaml

Expand All @@ -21,6 +23,8 @@
datefmt="%H:%M:%S",
)

_start_time: datetime | None = None


@dataclass
class Config:
Expand Down Expand Up @@ -413,10 +417,18 @@ def get_submitted(config: Config):
required=False,
type=click.Path(exists=True),
)
def submit_to_loculus(metadata, sequences, mode, log_level, config_file, output, revoke_map):
@click.option(
"--approve-timeout",
required=False,
type=int,
)
def submit_to_loculus(
metadata, sequences, mode, log_level, config_file, output, revoke_map, approve_timeout
):
"""
Submit data to Loculus.
"""
global _start_time
logger.setLevel(log_level)
logging.getLogger("requests").setLevel(logging.WARNING)
logging.getLogger("urllib3").setLevel(logging.WARNING)
Expand Down Expand Up @@ -449,10 +461,14 @@ def record_factory(*args, **kwargs):

if mode == "approve":
while True:
if not _start_time:
_start_time = datetime.now(tz=pytz.utc)
logger.info("Approving sequences")
response = approve(config)
logger.info(f"Approved: {len(response)} sequences")
sleep(30)
if datetime.now(tz=pytz.utc) - timedelta(minutes=approve_timeout) > _start_time:
break

if mode == "regroup-and-revoke":
try:
Expand Down

0 comments on commit 8b932a5

Please sign in to comment.