Skip to content
This repository has been archived by the owner on Sep 5, 2024. It is now read-only.

Commit

Permalink
some lint fixes
Browse files Browse the repository at this point in the history
  • Loading branch information
michaelharms committed Jan 18, 2020
1 parent 55dee60 commit 3816a2a
Show file tree
Hide file tree
Showing 5 changed files with 12 additions and 8 deletions.
2 changes: 1 addition & 1 deletion comcrawl/core/index_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -75,4 +75,4 @@ def download_pages(self, threads: int = None) -> None:
multi-threading only if set.
"""
self.results = download_multiple_results(self.results)
self.results = download_multiple_results(self.results, threads)
5 changes: 1 addition & 4 deletions comcrawl/types/types.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,9 @@
"""

from typing import List, Dict, Tuple
from typing import List, Dict

IndexList = List[str]

Result = Dict
ResultList = List[Dict]

HTMLStr = str
HTMLStrList = List[Tuple[str, str]]
5 changes: 3 additions & 2 deletions comcrawl/utils/download.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
import io
import gzip
import requests
from ..types import Result, ResultList, HTMLStr
from ..types import Result, ResultList
from .multithreading import make_multithreaded


Expand Down Expand Up @@ -51,7 +51,8 @@ def download_single_result(result: Result) -> Result:
return result


def download_multiple_results(results: ResultList, threads: int = None) -> ResultList:
def download_multiple_results(results: ResultList,
threads: int = None) -> ResultList:
"""Downloads search results.
For each Common Crawl search result in the given list the
Expand Down
7 changes: 7 additions & 0 deletions comcrawl/utils/multithreading.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,10 @@
"""Multithreading Helpers
This module contains utility functions
to manage multi-threading.
"""

from typing import Callable, List
from concurrent import futures

Expand Down
1 change: 0 additions & 1 deletion comcrawl/utils/search.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@
"""

from concurrent import futures
import json
import requests
from ..types import ResultList, IndexList
Expand Down

0 comments on commit 3816a2a

Please sign in to comment.