diff --git a/test/test_stock.py b/test/test_stock.py index ea4c7b6..92fbbd2 100644 --- a/test/test_stock.py +++ b/test/test_stock.py @@ -23,6 +23,20 @@ def test_make_datatuple(self): self.assertEqual(dt.change, 2.0) self.assertEqual(dt.transaction, 15718) + def test_make_datatuple_without_prices(self): + data = ['106/05/02', '45,851,963', '9,053,856,108', '--', + '--', '--', '--', ' 0.00', '15,718'] + dt = self.fetcher._make_datatuple(data) + self.assertEqual(dt.date, datetime.datetime(2017, 5, 2)) + self.assertEqual(dt.capacity, 45851963) + self.assertEqual(dt.turnover, 9053856108) + self.assertEqual(dt.open, None) + self.assertEqual(dt.high, None) + self.assertEqual(dt.low, None) + self.assertEqual(dt.close, None) + self.assertEqual(dt.change, 0.0) + self.assertEqual(dt.transaction, 15718) + class TWSEFetcerTest(unittest.TestCase, FetcherTest): fetcher = stock.TWSEFetcher() @@ -33,7 +47,7 @@ class TPEXFetcherTest(unittest.TestCase, FetcherTest): def test_make_datatuple(self): data = ['106/05/02', '45,851', '9,053,856', '198.50', - '199.00', '195.50', '196.50', '+2.00', '15,718'] + '199.00', '195.50', '196.50', '2.00', '15,718'] dt = self.fetcher._make_datatuple(data) self.assertEqual(dt.date, datetime.datetime(2017, 5, 2)) self.assertEqual(dt.capacity, 45851000) @@ -45,6 +59,20 @@ def test_make_datatuple(self): self.assertEqual(dt.change, 2.0) self.assertEqual(dt.transaction, 15718) + def test_make_datatuple_without_prices(self): + data = ['106/05/02', '45,851', '9,053,856', '--', + '--', '--', '--', '0.00', '15,718'] + dt = self.fetcher._make_datatuple(data) + self.assertEqual(dt.date, datetime.datetime(2017, 5, 2)) + self.assertEqual(dt.capacity, 45851000) + self.assertEqual(dt.turnover, 9053856000) + self.assertEqual(dt.open, None) + self.assertEqual(dt.high, None) + self.assertEqual(dt.low, None) + self.assertEqual(dt.close, None) + self.assertEqual(dt.change, 0.0) + self.assertEqual(dt.transaction, 15718) + class StockTest(object): def test_fetch_31(self): diff --git a/twstock/stock.py b/twstock/stock.py index 842c4ff..ba2f706 100644 --- a/twstock/stock.py +++ b/twstock/stock.py @@ -1,10 +1,12 @@ # -*- coding: utf-8 -*- import datetime -import json import urllib.parse from collections import namedtuple -import sys +try: + from json.decoder import JSONDecodeError +except ImportError: + JSONDecodeError = ValueError import requests @@ -43,23 +45,19 @@ class TWSEFetcher(BaseFetcher): def __init__(self): pass - def fetch(self, year: int, month: int, sid: str, retry=5): + def fetch(self, year: int, month: int, sid: str, retry: int=5): params = {'date': '%d%02d01' % (year, month), 'stockNo': sid} - r = requests.get(self.REPORT_URL, params=params) - if sys.version_info < (3, 5): + for retry_i in range(retry): + r = requests.get(self.REPORT_URL, params=params) try: data = r.json() - except ValueError: - if retry: - return self.fetch(year, month, sid, retry - 1) - data = {'stat': '', 'data': []} + except JSONDecodeError: + continue + else: + break else: - try: - data = r.json() - except json.decoder.JSONDecodeError: - if retry: - return self.fetch(year, month, sid, retry - 1) - data = {'stat': '', 'data': []} + # Fail in all retries + data = {'stat': '', 'data': []} if data['stat'] == 'OK': data['data'] = self.purify(data) @@ -71,16 +69,17 @@ def _make_datatuple(self, data): data[0] = datetime.datetime.strptime(self._convert_date(data[0]), '%Y/%m/%d') data[1] = int(data[1].replace(',', '')) data[2] = int(data[2].replace(',', '')) - data[3] = float(data[3].replace(',', '')) - data[4] = float(data[4].replace(',', '')) - data[5] = float(data[5].replace(',', '')) - data[6] = float(data[6].replace(',', '')) - data[7] = float(0.0 if data[7].replace(',', '') == 'X0.00' else data[7].replace(',', '')) # +/-/X表示漲/跌/不比價 + data[3] = None if data[3] == '--' else float(data[3].replace(',', '')) + data[4] = None if data[4] == '--' else float(data[4].replace(',', '')) + data[5] = None if data[5] == '--' else float(data[5].replace(',', '')) + data[6] = None if data[6] == '--' else float(data[6].replace(',', '')) + # +/-/X表示漲/跌/不比價 + data[7] = float(0.0 if data[7].replace(',', '') == 'X0.00' else data[7].replace(',', '')) data[8] = int(data[8].replace(',', '')) return DATATUPLE(*data) def purify(self, original_data): - return [self._make_datatuple(d) for d in original_data['data'] if d[3] != '--'] + return [self._make_datatuple(d) for d in original_data['data']] class TPEXFetcher(BaseFetcher): @@ -90,10 +89,19 @@ class TPEXFetcher(BaseFetcher): def __init__(self): pass - def fetch(self, year: int, month: int, sid: str): + def fetch(self, year: int, month: int, sid: str, retry: int=5): params = {'d': '%d/%d' % (year - 1911, month), 'stkno': sid} - r = requests.get(self.REPORT_URL, params=params) - data = r.json() + for retry_i in range(retry): + r = requests.get(self.REPORT_URL, params=params) + try: + data = r.json() + except JSONDecodeError: + continue + else: + break + else: + # Fail in all retries + data = {'aaData': []} data['data'] = [] if data['aaData']: @@ -108,10 +116,10 @@ def _make_datatuple(self, data): data[0] = datetime.datetime.strptime(self._convert_date(data[0]), '%Y/%m/%d') data[1] = int(data[1].replace(',', '')) * 1000 data[2] = int(data[2].replace(',', '')) * 1000 - data[3] = float(data[3].replace(',', '')) - data[4] = float(data[4].replace(',', '')) - data[5] = float(data[5].replace(',', '')) - data[6] = float(data[6].replace(',', '')) + data[3] = None if data[3] == '--' else float(data[3].replace(',', '')) + data[4] = None if data[4] == '--' else float(data[4].replace(',', '')) + data[5] = None if data[5] == '--' else float(data[5].replace(',', '')) + data[6] = None if data[6] == '--' else float(data[6].replace(',', '')) data[7] = float(data[7].replace(',', '')) data[8] = int(data[8].replace(',', '')) return DATATUPLE(*data)