Merge pull request #7 from fasiondog/master

update 20221017

.github/workflows/ubuntu.yml

@@ -24,7 +24,7 @@ jobs:
         ls
         ./bootstrap.sh --with-python=python3
         ./b2 release link=shared address-model=64 -j 4 --with-python --with-serialization; 
-        ./b2 release link=static address-model=64 cxxflags=-fPIC -j 4 --with-date_time --with-filesystem --with-system --with-test; 
+        ./b2 release link=static address-model=64 cxxflags=-fPIC -j 4 --with-date_time --with-filesystem --with-system --with-test --with-atomic; 
       
     - name: test
       run: |

.gitignore

@@ -22,6 +22,7 @@
 .vscode
 .vs
 .ipynb_checkpoints
+nohup.out
 temp
 file
 temp.txt

docs/source/base/constant.rst

@@ -78,5 +78,7 @@ Null 值及证券类别
     .. py:attribute:: STOCKTYPE_BOND 股票类型-其他债券
     
     .. py:attribute:: STOCKTYPE_GEM 股票类型-创业板
+
+    .. py:attribute:: STOCKTYPE_START 股票类型-创业板
     
     .. py:attribute:: STOCKTYPE_TMP 股票类型-临时CSV

docs/source/quickstart.rst

@@ -146,15 +146,15 @@ matplotlib显示图形时中文字体乱码的问题
 
     #font.sans-serif     : DejaVu Sans, Bitstream Vera Sans, Lucida Grande, Verdana, Geneva, Lucid, Arial, Helvetica, Avant Garde, sans-serif
 
-注释去掉，在冒号后面添加 SimHei，可以正常显示中文标签。
+注释去掉，在冒号后面添加 SimHei（如果使用 Ubuntu，可以使用 “Noto Sans CJK JP”），可以正常显示中文标签。
 
 再把下面一行的注释去掉，把冒号后面改成False可以正常显示正负号。
 
 ::
 
     #axes.unicode_minus  : True
     
-2. 删除 “.matplotlib” 目录下的字体缓存文件 “fontList.py3k.cache”
+2. 删除 “.matplotlib” 目录下的字体缓存文件 “fontList.py3k.cache”。（Ubuntu 下，该位置为用户目录 .cache/matplotlib，删除该目录下所有文件）
 
 3. 检查 “c:\\windows\\fons” 目录下，是否存在 simhei.ttf 文件，如果没有可从网上搜索下载或加入Hikyuu QQ群下载字体文件，将该字体文件拷贝至 “c:\\windows\\fons” 目录下。
 

docs/source/release.rst

@@ -1,6 +1,26 @@
 版本发布说明
 =======================
 
+1.2.5 - 2022年9月3日
+-------------------------
+
+1. 增加北京交易所数据
+2. 改进数据下载，修复 pytdx 数据下载缺失部分数据
+3. 恢复财务数据下载
+4. 增加 start_insight_sdk.py, 从华泰 insight 获取实时数据
+5. 完善 hikyuuTdx 中 nng 消息的启停与释放
+6. hku_catch 增加指示重新抛出异常的参数
+7. 修正 demo
+
+
+1.2.4 - 2022年6月30日
+-------------------------
+
+1. 修复 trade_manage持久化，费率设置为TC_FixedA2017会造成持久化中断 
+2. 修改 TradeManager::getFunds 中的截止时间 23:59 分被误写为 11:59 分
+3. 修复订单代理失效
+
+
 1.2.3 - 2022年3月6日
 -------------------------
 

docs/source/stock_manager.rst

@@ -145,6 +145,10 @@ StockManager/Block/Stock
     .. py:method:: tmpdir(self)
     
         获取用于保存零时变量等的临时目录，如未配置则为当前目录 由m_config中的“tmpdir”指定
+
+    .. py:method:: datadir(self)
+
+        获取财务数据目录
 
     .. py:method:: get_market_list(self)
     

docs/source/trade_manage/OrderBroker.rst

@@ -18,12 +18,11 @@ Python中的订单代理包装
     my_tm = crtTM(init_cash = 300000)
 
     #注册实盘交易订单代理
-    my_tm.regBroker(crtOB(TestOrderBroker())) #TestOerderBroker是测试用订单代理对象，只打印
-    #my_tm.regBroker(crtOB(Puppet()))  #Puppet为内建的扯线木偶实盘下单对象
-    #my_tm.regBroker(crtOB(MailOrderBroker("smtp.sina.com", "[email protected]", "yourpwd", "[email protected])))
+    my_tm.reg_broker(crtOB(TestOrderBroker())) #TestOerderBroker是测试用订单代理对象，只打印
+    #my_tm.reg_broker(crtOB(MailOrderBroker("smtp.sina.com", "[email protected]", "yourpwd", "[email protected])))
 
     #根据需要修改订单代理最后的时间戳，后续只有大于该时间戳时，订单代理才会实际发出订单指令
-    my_tm.brokeLastDatetime=Datetime(201706010000)
+    my_tm.broke_last_datetime=Datetime(201706010000)
 
     #创建信号指示器（以5日EMA为快线，5日EMA自身的10日EMA作为慢线，快线向上穿越慢线时买入，反之卖出）
     my_sg = SG_Flex(OP(EMA(n=5)), slow_n=10)

docs/source/trade_sys/system.rst

@@ -8,7 +8,8 @@
 
 公共参数：
 
-    * **delay=True** *(bool)* : 是否延迟到下一个bar开盘时进行交易
+    * **buy_delay=True** *(bool)* : 买入操作是否延迟到下一个bar开盘时进行交易
+    * **sell_delay=True** *(bool)* : 卖出操作是否延迟到下一个bar开盘时进行交易
     * **delay_use_current_price=True** *(bool)* : 延迟操作的情况下，是使用当前交易时bar的价格计算新的止损价/止赢价/目标价还是使用上次计算的结果
     * **max_delay_count=3** *(int)* : 连续延迟交易请求的限制次数，应大于等于0，0表示只允许延迟1次
     * **tp_monotonic=True** *(bool)* : 止赢单调递增

hikyuu/admin/data/LocalDatabase.py

@@ -9,7 +9,10 @@ class LocalDatabase:
     model = declarative_base()
 
     def __init__(self):
-        self._db_name = "%s/.hikyuu/admin/hkuadmin.db" % os.path.expanduser('~')
+        path = "%s/.hikyuu/admin" % os.path.expanduser('~')
+        if not os.path.lexists(path):
+            os.makedirs(path)
+        self._db_name = f"{path}/hkuadmin.db"
         self._engine = sqlalchemy.create_engine(
             "sqlite:///%s?check_same_thread=false" % self._db_name, echo=True, future=True
         )

hikyuu/data/common.py

@@ -22,15 +22,27 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
 
+import requests
+import re
+import akshare as ak
+import pandas as pd
+import datetime
+from hikyuu.util import *
+
 
 class MARKET:
     SH = 'SH'
     SZ = 'SZ'
+    BJ = 'BJ'
+
+
+g_market_list = [MARKET.SH, MARKET.SZ, MARKET.BJ]
 
 
 class MARKETID:
     SH = 1
     SZ = 2
+    BJ = 3
 
 
 class STOCKTYPE:
@@ -69,4 +81,94 @@ def get_stktype_list(quotations=None):
         else:
             print('Unknow quotation: {}'.format(quotation))
 
-    return tuple(result)
+    return tuple(result)
+
+
+@hku_catch(ret=[], trace=True)
+def get_stk_code_name_list(market: str) -> list:
+    """
+    获取指定证券交易所股票代码与名称列表
+    :return: 代码名称对组成的列表：[{'code': 'code1': 'name': 'name1'}, ...]
+    """
+    # 获取深圳股票代码表
+    if market == MARKET.SZ:
+        ind_list = ["A股列表", "B股列表"]
+        df = None
+        for ind in ind_list:
+            tmp_df = ak.stock_info_sz_name_code(ind)
+            tmp_df.rename(columns={'A股代码': 'code', 'A股简称': 'name'}, inplace=True)
+            df = pd.concat([df, tmp_df]) if df is not None else tmp_df
+        hku_info("获取深圳证券交易所股票数量: {}", len(df) if df is not None else 0)
+        return df[['code', 'name']].to_dict(orient='records') if df is not None else []
+
+    # 获取上证股票代码表
+    if market == MARKET.SH:
+        ind_list = ["主板A股", "主板B股", "科创板"]
+        df = None
+        for ind in ind_list:
+            tmp_df = ak.stock_info_sh_name_code(ind)
+            tmp_df.rename(columns={'证券代码': 'code', '证券简称': 'name'}, inplace=True)
+            df = pd.concat([df, tmp_df]) if df is not None else tmp_df
+        hku_info("获取上海证券交易所股票数量: {}", len(df) if df is not None else 0)
+        return df[['code', 'name']].to_dict(orient='records') if df is not None else []
+
+    # 获取北京股票代码表
+    if market == MARKET.BJ:
+        df = ak.stock_info_bj_name_code()
+        df.rename(columns={'证券代码': 'code', '证券简称': 'name'}, inplace=True)
+        hku_info("获取北京证券交易所股票数量: {}", len(df) if df is not None else 0)
+        return df[['code', 'name']].to_dict(orient='records') if df is not None else []
+
+
+@hku_catch(ret=[], trace=True)
+def get_index_code_name_list() -> list:
+    """
+    获取所有股票指数代码名称列表
+    从新浪获取，多次频繁调用会被封禁IP，需10分钟后再试
+
+    :return: [{'market_code': 'SHxxx'}, ...]
+    """
+    df = ak.stock_zh_index_spot()
+    return [{'market_code': df.loc[i]['代码'].upper(), 'name': df.loc[i]['名称']} for i in range(len(df))]
+
+
+g_fund_code_name_list = {}
+for market in g_market_list:
+    g_fund_code_name_list[market] = []
+g_last_get_fund_code_name_list_date = datetime.date(1990, 12, 9)
+
+
+@hku_catch(ret=[], trace=True)
+def get_fund_code_name_list(market: str) -> list:
+    """
+    获取基金代码名称列表 (来源: sina)
+    """
+    # 保证一天只获取一次基金股票代码表，防止对 sina 的频繁访问
+    global g_last_get_fund_code_name_list_date
+    now = datetime.date.today()
+    if now <= g_last_get_fund_code_name_list_date:
+        return g_fund_code_name_list[market]
+
+    ind_list = "封闭式基金", "ETF基金", "LOF基金"
+    for ind in ind_list:
+        df = ak.fund_etf_category_sina(ind)
+        for i in range(len(df)):
+            loc = df.loc[i]
+            try:
+                code, name = str(loc['代码']), str(loc['名称'])
+                g_fund_code_name_list[code[:2].upper()].append(dict(code=code[2:], name=name))
+            except Exception as e:
+                hku_error("{}! {}", str(e), loc)
+    hku_info("获取基金列表数量: {}", len(g_fund_code_name_list[market]))
+    g_last_get_fund_code_name_list_date = now
+    return g_fund_code_name_list[market]
+
+
+@hku_catch(ret=[], trace=True)
+def get_new_holidays():
+    """获取新的交易所休假日历"""
+    res = requests.get('https://www.tdx.com.cn/url/holiday/')
+    res.encoding = res.apparent_encoding
+    ret = re.findall(r'<textarea id="data" style="display:none;">([\s\w\d\W]+)</textarea>', res.text, re.M)[0].strip()
+    day = [d.split('|')[:4] for d in ret.split('\n')]
+    return [v[0] for v in day if v[2] == '中国']

hikyuu/data/common_mysql.py

@@ -28,7 +28,7 @@
 
 import mysql.connector
 
-from hikyuu.data.common import MARKETID, get_stktype_list
+from hikyuu.data.common import get_stktype_list, get_new_holidays
 from hikyuu.util import hku_debug
 
 
@@ -116,6 +116,20 @@ def get_stock_list(connect, market, quotations):
     return a
 
 
+def import_new_holidays(connect):
+    """导入新的交易所休假日历"""
+    cur = connect.cursor()
+    cur.execute("select date from `hku_base`.`holiday` order by date desc limit 1")
+    a = cur.fetchall()
+    last_date = a[0][0] if a else 19901219
+    holidays = get_new_holidays()
+    new_holidays = [(int(v), ) for v in holidays if int(v) > last_date]
+    if new_holidays:
+        cur.executemany("insert into `hku_base`.`holiday` (date) values (%s)", new_holidays)
+        connect.commit()
+        cur.close()
+
+
 def get_table(connect, market, code, ktype):
     cur = connect.cursor()
     ktype_dict = {

hikyuu/data/common_pytdx.py

@@ -27,6 +27,24 @@
 from pytdx.hq import TdxHq_API
 from pytdx.config.hosts import hq_hosts
 
+hq_hosts = [
+    ('上海双线主站1', '47.103.48.45', 7709),
+    ('上海双线主站2', '47.103.86.229', 7709),
+    ('上海双线主站3', '47.103.88.146', 7709),
+    ('深圳双线主站1', '120.79.60.82', 7709),
+    ('深圳双线主站2', '47.112.129.66', 7709),
+    ('北京双线主站1', '39.98.234.173', 7709),
+    ('北京双线主站2', '39.98.198.249', 7709),
+    ('北京双线主站3', '39.100.68.59', 7709),
+]
+
+
+def to_pytdx_market(market):
+    """转换为pytdx的market"""
+    pytdx_market = {'SZ': 0, 'SH': 1, 'BJ': 2}
+    return pytdx_market[market.upper()]
+
+
 def ping(ip, port=7709, multithread=False):
     api = TdxHq_API(multithread=multithread)
     success = False

hikyuu/data/common_sqlite3.py

@@ -26,7 +26,7 @@
 import sqlite3
 from pathlib import Path
 
-from .common import MARKETID, get_stktype_list
+from .common import MARKETID, get_stktype_list, get_new_holidays
 
 
 def is_exist_db(connect):
@@ -124,4 +124,17 @@ def get_stock_list(connect, market, quotations):
     a = cur.execute(sql).fetchall()
     connect.commit()
     cur.close()
-    return a
+    return a
+
+
+def import_new_holidays(connect):
+    """导入新的交易所休假日历"""
+    cur = connect.cursor()
+    a = cur.execute("select date from holiday order by date desc limit 1").fetchall()
+    last_date = a[0][0] if a else 19901219
+    holidays = get_new_holidays()
+    new_holidays = [(int(v), ) for v in holidays if int(v) > last_date]
+    if new_holidays:
+        cur.executemany("insert into holiday (date) values (?)", new_holidays)
+        connect.commit()
+        cur.close()

hikyuu/data/hku_config_template.py

@@ -79,6 +79,11 @@
 sz_time = {dir}/sz_time.h5
 sh_trans = {dir}/sh_trans.h5
 sz_trans = {dir}/sz_trans.h5
+bj_day = {dir}/bj_day.h5
+bj_min = {dir}/bj_1min.h5
+bj_min5 = {dir}/bj_5min.h5
+bj_time = {dir}/bj_time.h5
+bj_trans = {dir}/bj_trans.h5
 """
 
 mysql_template = """

hikyuu/data/mysql_upgrade/0004.sql

@@ -1,26 +1,26 @@
-CREATE TABLE IF NOT EXISTS `hku_base`.`Holiday` (
+CREATE TABLE IF NOT EXISTS `hku_base`.`holiday` (
 	`id`	INT UNSIGNED NOT NULL AUTO_INCREMENT,
 	`date` BIGINT UNSIGNED NOT NULL,
 	PRIMARY KEY(`id`)
 );
-INSERT INTO `hku_base`.`Holiday` (`date`) VALUES (20210101);
-INSERT INTO `hku_base`.`Holiday` (`date`) VALUES (20210211);
-INSERT INTO `hku_base`.`Holiday` (`date`) VALUES (20210212);
-INSERT INTO `hku_base`.`Holiday` (`date`) VALUES (20210215);
-INSERT INTO `hku_base`.`Holiday` (`date`) VALUES (20210216);
-INSERT INTO `hku_base`.`Holiday` (`date`) VALUES (20210217);
-INSERT INTO `hku_base`.`Holiday` (`date`) VALUES (20210405);
-INSERT INTO `hku_base`.`Holiday` (`date`) VALUES (20210501);
-INSERT INTO `hku_base`.`Holiday` (`date`) VALUES (20210503);
-INSERT INTO `hku_base`.`Holiday` (`date`) VALUES (20210504);
-INSERT INTO `hku_base`.`Holiday` (`date`) VALUES (20210505);
-INSERT INTO `hku_base`.`Holiday` (`date`) VALUES (20210614);
-INSERT INTO `hku_base`.`Holiday` (`date`) VALUES (20210920);
-INSERT INTO `hku_base`.`Holiday` (`date`) VALUES (20210921);
-INSERT INTO `hku_base`.`Holiday` (`date`) VALUES (20211001);
-INSERT INTO `hku_base`.`Holiday` (`date`) VALUES (20211004);
-INSERT INTO `hku_base`.`Holiday` (`date`) VALUES (20211005);
-INSERT INTO `hku_base`.`Holiday` (`date`) VALUES (20211006);
-INSERT INTO `hku_base`.`Holiday` (`date`) VALUES (20211007);
+INSERT INTO `hku_base`.`holiday` (`date`) VALUES (20210101);
+INSERT INTO `hku_base`.`holiday` (`date`) VALUES (20210211);
+INSERT INTO `hku_base`.`holiday` (`date`) VALUES (20210212);
+INSERT INTO `hku_base`.`holiday` (`date`) VALUES (20210215);
+INSERT INTO `hku_base`.`holiday` (`date`) VALUES (20210216);
+INSERT INTO `hku_base`.`holiday` (`date`) VALUES (20210217);
+INSERT INTO `hku_base`.`holiday` (`date`) VALUES (20210405);
+INSERT INTO `hku_base`.`holiday` (`date`) VALUES (20210501);
+INSERT INTO `hku_base`.`holiday` (`date`) VALUES (20210503);
+INSERT INTO `hku_base`.`holiday` (`date`) VALUES (20210504);
+INSERT INTO `hku_base`.`holiday` (`date`) VALUES (20210505);
+INSERT INTO `hku_base`.`holiday` (`date`) VALUES (20210614);
+INSERT INTO `hku_base`.`holiday` (`date`) VALUES (20210920);
+INSERT INTO `hku_base`.`holiday` (`date`) VALUES (20210921);
+INSERT INTO `hku_base`.`holiday` (`date`) VALUES (20211001);
+INSERT INTO `hku_base`.`holiday` (`date`) VALUES (20211004);
+INSERT INTO `hku_base`.`holiday` (`date`) VALUES (20211005);
+INSERT INTO `hku_base`.`holiday` (`date`) VALUES (20211006);
+INSERT INTO `hku_base`.`holiday` (`date`) VALUES (20211007);
 
 UPDATE `hku_base`.`version` set `version` = 4;

hikyuu/data/mysql_upgrade/0005.sql

@@ -1,3 +1,3 @@
-INSERT INTO `hku_base`.`StockTypeInfo` (`id`, `tickValue`,`precision`,`type`,`description`,`tick`,`minTradeNumber`,`maxTradeNumber`) VALUES (9, 0.01,2,9,'科创板',0.01,100,1000000);
-INSERT INTO `hku_base`.`CodeRuleType` (`id`,`marketid`,`codepre`,`type`,`description`) VALUES (33, 1,'688',9,'科创板');
+INSERT INTO `hku_base`.`stocktypeinfo` (`id`, `tickValue`,`precision`,`type`,`description`,`tick`,`minTradeNumber`,`maxTradeNumber`) VALUES (9, 0.01,2,9,'科创板',0.01,100,1000000);
+INSERT INTO `hku_base`.`coderuletype` (`id`,`marketid`,`codepre`,`type`,`description`) VALUES (33, 1,'688',9,'科创板');
 UPDATE `hku_base`.`version` set `version` = 5;