diff --git a/README.md b/README.md index 8adde52..235cb44 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ from comcrawl import IndexClient client = IndexClient() client.search("reddit.com/r/MachineLearning/*") -client.download_pages() +client.download() results = client.results ``` @@ -59,7 +59,7 @@ from comcrawl import IndexClient client = IndexClient() client.search("reddit.com/r/MachineLearning/*", threads=4) -client.download_pages() +client.download() results = client.results ``` @@ -80,7 +80,7 @@ sorted_df = df.sort_values(by="timestamp") filtered_df = sorted_df.drop_duplicates("urlkey", keep="last") client.results = filtered_df.to_dict("records") -client.download_pages() +client.download() pd.DataFrame(client.results).to_csv("results.csv") ``` @@ -94,7 +94,7 @@ from comcrawl import IndexClient client = IndexClient(["2019-51", "2019-47"]) client.search("reddit.com/r/MachineLearning/*") -client.download_pages() +client.download() results = client.results ``` @@ -108,7 +108,7 @@ from comcrawl import IndexClient client = IndexClient(verbose=True) client.search("reddit.com/r/MachineLearning/*") -client.download_pages() +client.download() results = client.results ``` diff --git a/comcrawl/core/index_client.py b/comcrawl/core/index_client.py index 1c4adb2..1ca2fab 100644 --- a/comcrawl/core/index_client.py +++ b/comcrawl/core/index_client.py @@ -64,7 +64,7 @@ def search(self, url: str, threads: int = None) -> None: """ self.results = search_multiple_indexes(url, self.indexes, threads) - def download_pages(self, threads: int = None) -> None: + def download(self, threads: int = None) -> None: """Download Downloads the HTML for every result in the diff --git a/tests/test_comcrawl.py b/tests/test_comcrawl.py index 664e894..47b0cde 100644 --- a/tests/test_comcrawl.py +++ b/tests/test_comcrawl.py @@ -17,6 +17,6 @@ def test_comcrawl(snapshot): assert len(client.results) == 2 - client.download_pages() + client.download() snapshot.assert_match(client.results[1])