Merge pull request #42 from cullenwatson/fix/class-type-error

- refactor & #41 bug fix

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "python-jobspy"
-version = "1.1.1"
+version = "1.1.2"
 description = "Job scraper for LinkedIn, Indeed & ZipRecruiter"
 authors = ["Zachary Hampton <[email protected]>", "Cullen Watson <[email protected]>"]
 readme = "README.md"

src/jobspy/__init__.py

@@ -26,7 +26,7 @@ def _map_str_to_site(site_name: str) -> Site:
 
 
 def scrape_jobs(
-    site_name: str | Site | List[Site],
+    site_name: str | List[str] | Site | List[Site],
     search_term: str,
     location: str = "",
     distance: int = None,
@@ -43,11 +43,12 @@ def scrape_jobs(
     """
 
     if type(site_name) == str:
-        site_name = _map_str_to_site(site_name)
+        site_type = [_map_str_to_site(site_name)]
+    else:  #: if type(site_name) == list
+        site_type = [_map_str_to_site(site) if type(site) == str else site_name for site in site_name]
 
     country_enum = Country.from_string(country_indeed)
 
-    site_type = [site_name] if type(site_name) == Site else site_name
     scraper_input = ScraperInput(
         site_type=site_type,
         country=country_enum,
@@ -122,7 +123,6 @@ def worker(site):
     errors_list = [(key, value) for key, value in errors.items()]
     errors_df = pd.DataFrame(errors_list, columns=["Site", "Error"])
 
-
     if dfs:
         df = pd.concat(dfs, ignore_index=True)
         if hyperlinks:

src/jobspy/scrapers/indeed/__init__.py

@@ -78,9 +78,7 @@ def scrape_page(
             raise StatusException(response.status_code)
 
         soup = BeautifulSoup(response.content, "html.parser")
-        with open("text2.html", "w", encoding="utf-8") as f:
-            f.write(str(soup))
-        if "did not match any jobs" in str(soup):
+        if "did not match any jobs" in response.text:
             raise ParsingException("Search did not match any jobs")
 
         jobs = IndeedScraper.parse_jobs(

src/tests/test_all.py

@@ -0,0 +1,10 @@
+from ..jobspy import scrape_jobs
+
+
+def test_all():
+    result = scrape_jobs(
+        site_name=["linkedin", "indeed", "zip_recruiter"],
+        search_term="software engineer",
+        results_wanted=5,
+    )
+    assert result is not None and result.errors.empty is True

src/tests/test_indeed.py

@@ -6,4 +6,4 @@ def test_indeed():
         site_name="indeed",
         search_term="software engineer",
     )
-    assert result is not None
+    assert result is not None and result.errors.empty is True

src/tests/test_linkedin.py

@@ -1,9 +1,9 @@
-from jobspy import scrape_jobs
+from ..jobspy import scrape_jobs
 
 
 def test_linkedin():
     result = scrape_jobs(
         site_name="linkedin",
         search_term="software engineer",
     )
-    assert result is not None
+    assert result is not None and result.errors.empty is True

src/tests/test_ziprecruiter.py

@@ -1,4 +1,4 @@
-from jobspy import scrape_jobs
+from ..jobspy import scrape_jobs
 
 
 def test_ziprecruiter():
@@ -7,4 +7,4 @@ def test_ziprecruiter():
         search_term="software engineer",
     )
 
-    assert result is not None
+    assert result is not None and result.errors.empty is True