Skip to content

Commit

Permalink
Replace '--' with None (fixed #27) (#35)
Browse files Browse the repository at this point in the history
* Unify the retry behavior in TWSEFetcher and TPEXFetcher, and improve coding style

* Replace '--' with None

* Add unit test for data with prices '--'
  • Loading branch information
ianlini authored and mlouielu committed Mar 16, 2018
1 parent 80320e0 commit 91c1c75
Show file tree
Hide file tree
Showing 2 changed files with 65 additions and 29 deletions.
30 changes: 29 additions & 1 deletion test/test_stock.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,20 @@ def test_make_datatuple(self):
self.assertEqual(dt.change, 2.0)
self.assertEqual(dt.transaction, 15718)

def test_make_datatuple_without_prices(self):
data = ['106/05/02', '45,851,963', '9,053,856,108', '--',
'--', '--', '--', ' 0.00', '15,718']
dt = self.fetcher._make_datatuple(data)
self.assertEqual(dt.date, datetime.datetime(2017, 5, 2))
self.assertEqual(dt.capacity, 45851963)
self.assertEqual(dt.turnover, 9053856108)
self.assertEqual(dt.open, None)
self.assertEqual(dt.high, None)
self.assertEqual(dt.low, None)
self.assertEqual(dt.close, None)
self.assertEqual(dt.change, 0.0)
self.assertEqual(dt.transaction, 15718)


class TWSEFetcerTest(unittest.TestCase, FetcherTest):
fetcher = stock.TWSEFetcher()
Expand All @@ -33,7 +47,7 @@ class TPEXFetcherTest(unittest.TestCase, FetcherTest):

def test_make_datatuple(self):
data = ['106/05/02', '45,851', '9,053,856', '198.50',
'199.00', '195.50', '196.50', '+2.00', '15,718']
'199.00', '195.50', '196.50', '2.00', '15,718']
dt = self.fetcher._make_datatuple(data)
self.assertEqual(dt.date, datetime.datetime(2017, 5, 2))
self.assertEqual(dt.capacity, 45851000)
Expand All @@ -45,6 +59,20 @@ def test_make_datatuple(self):
self.assertEqual(dt.change, 2.0)
self.assertEqual(dt.transaction, 15718)

def test_make_datatuple_without_prices(self):
data = ['106/05/02', '45,851', '9,053,856', '--',
'--', '--', '--', '0.00', '15,718']
dt = self.fetcher._make_datatuple(data)
self.assertEqual(dt.date, datetime.datetime(2017, 5, 2))
self.assertEqual(dt.capacity, 45851000)
self.assertEqual(dt.turnover, 9053856000)
self.assertEqual(dt.open, None)
self.assertEqual(dt.high, None)
self.assertEqual(dt.low, None)
self.assertEqual(dt.close, None)
self.assertEqual(dt.change, 0.0)
self.assertEqual(dt.transaction, 15718)


class StockTest(object):
def test_fetch_31(self):
Expand Down
64 changes: 36 additions & 28 deletions twstock/stock.py
Original file line number Diff line number Diff line change
@@ -1,10 +1,12 @@
# -*- coding: utf-8 -*-

import datetime
import json
import urllib.parse
from collections import namedtuple
import sys
try:
from json.decoder import JSONDecodeError
except ImportError:
JSONDecodeError = ValueError

import requests

Expand Down Expand Up @@ -43,23 +45,19 @@ class TWSEFetcher(BaseFetcher):
def __init__(self):
pass

def fetch(self, year: int, month: int, sid: str, retry=5):
def fetch(self, year: int, month: int, sid: str, retry: int=5):
params = {'date': '%d%02d01' % (year, month), 'stockNo': sid}
r = requests.get(self.REPORT_URL, params=params)
if sys.version_info < (3, 5):
for retry_i in range(retry):
r = requests.get(self.REPORT_URL, params=params)
try:
data = r.json()
except ValueError:
if retry:
return self.fetch(year, month, sid, retry - 1)
data = {'stat': '', 'data': []}
except JSONDecodeError:
continue
else:
break
else:
try:
data = r.json()
except json.decoder.JSONDecodeError:
if retry:
return self.fetch(year, month, sid, retry - 1)
data = {'stat': '', 'data': []}
# Fail in all retries
data = {'stat': '', 'data': []}

if data['stat'] == 'OK':
data['data'] = self.purify(data)
Expand All @@ -71,16 +69,17 @@ def _make_datatuple(self, data):
data[0] = datetime.datetime.strptime(self._convert_date(data[0]), '%Y/%m/%d')
data[1] = int(data[1].replace(',', ''))
data[2] = int(data[2].replace(',', ''))
data[3] = float(data[3].replace(',', ''))
data[4] = float(data[4].replace(',', ''))
data[5] = float(data[5].replace(',', ''))
data[6] = float(data[6].replace(',', ''))
data[7] = float(0.0 if data[7].replace(',', '') == 'X0.00' else data[7].replace(',', '')) # +/-/X表示漲/跌/不比價
data[3] = None if data[3] == '--' else float(data[3].replace(',', ''))
data[4] = None if data[4] == '--' else float(data[4].replace(',', ''))
data[5] = None if data[5] == '--' else float(data[5].replace(',', ''))
data[6] = None if data[6] == '--' else float(data[6].replace(',', ''))
# +/-/X表示漲/跌/不比價
data[7] = float(0.0 if data[7].replace(',', '') == 'X0.00' else data[7].replace(',', ''))
data[8] = int(data[8].replace(',', ''))
return DATATUPLE(*data)

def purify(self, original_data):
return [self._make_datatuple(d) for d in original_data['data'] if d[3] != '--']
return [self._make_datatuple(d) for d in original_data['data']]


class TPEXFetcher(BaseFetcher):
Expand All @@ -90,10 +89,19 @@ class TPEXFetcher(BaseFetcher):
def __init__(self):
pass

def fetch(self, year: int, month: int, sid: str):
def fetch(self, year: int, month: int, sid: str, retry: int=5):
params = {'d': '%d/%d' % (year - 1911, month), 'stkno': sid}
r = requests.get(self.REPORT_URL, params=params)
data = r.json()
for retry_i in range(retry):
r = requests.get(self.REPORT_URL, params=params)
try:
data = r.json()
except JSONDecodeError:
continue
else:
break
else:
# Fail in all retries
data = {'aaData': []}

data['data'] = []
if data['aaData']:
Expand All @@ -108,10 +116,10 @@ def _make_datatuple(self, data):
data[0] = datetime.datetime.strptime(self._convert_date(data[0]), '%Y/%m/%d')
data[1] = int(data[1].replace(',', '')) * 1000
data[2] = int(data[2].replace(',', '')) * 1000
data[3] = float(data[3].replace(',', ''))
data[4] = float(data[4].replace(',', ''))
data[5] = float(data[5].replace(',', ''))
data[6] = float(data[6].replace(',', ''))
data[3] = None if data[3] == '--' else float(data[3].replace(',', ''))
data[4] = None if data[4] == '--' else float(data[4].replace(',', ''))
data[5] = None if data[5] == '--' else float(data[5].replace(',', ''))
data[6] = None if data[6] == '--' else float(data[6].replace(',', ''))
data[7] = float(data[7].replace(',', ''))
data[8] = int(data[8].replace(',', ''))
return DATATUPLE(*data)
Expand Down

0 comments on commit 91c1c75

Please sign in to comment.