Merge pull request #414 from pikers/agg_feedz

Agg feedz

.github/workflows/ci.yml

@@ -14,6 +14,27 @@ on:
 
 jobs:
 
+  # test that we can generate a software distribution and install it
+  # thus avoid missing file issues after packaging.
+  sdist-linux:
+    name: 'sdist'
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Setup python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.10'
+
+      - name: Build sdist
+        run: python setup.py sdist --formats=zip
+
+      - name: Install sdist from .zips
+        run: python -m pip install dist/*.zip
+
   testing:
     name: 'install + test-suite'
     runs-on: ubuntu-latest

dockering/ib/docker-compose.yml

@@ -62,39 +62,39 @@ services:
       # - "127.0.0.1:4002:4002"
       # - "127.0.0.1:5900:5900"
 
-  ib_gw_live:
-    image: waytrade/ib-gateway:1012.2i
-    restart: always
-    network_mode: 'host'
+  # ib_gw_live:
+  #   image: waytrade/ib-gateway:1012.2i
+  #   restart: always
+  #   network_mode: 'host'
 
-    volumes:
-      - type: bind
-        source: ./jts_live.ini
-        target: /root/jts/jts.ini
-        # don't let ibc clobber this file for
-        # the main reason of not having a stupid
-        # timezone set..
-        read_only: true
+  #   volumes:
+  #     - type: bind
+  #       source: ./jts_live.ini
+  #       target: /root/jts/jts.ini
+  #       # don't let ibc clobber this file for
+  #       # the main reason of not having a stupid
+  #       # timezone set..
+  #       read_only: true
 
-      # force our own ibc config
-      - type: bind
-        source: ./ibc.ini
-        target: /root/ibc/config.ini
+  #     # force our own ibc config
+  #     - type: bind
+  #       source: ./ibc.ini
+  #       target: /root/ibc/config.ini
 
-      # force our noop script - socat isn't needed in host mode.
-      - type: bind
-        source: ./fork_ports_delayed.sh
-        target: /root/scripts/fork_ports_delayed.sh
+  #     # force our noop script - socat isn't needed in host mode.
+  #     - type: bind
+  #       source: ./fork_ports_delayed.sh
+  #       target: /root/scripts/fork_ports_delayed.sh
 
-      # force our noop script - socat isn't needed in host mode.
-      - type: bind
-        source: ./run_x11_vnc.sh
-        target: /root/scripts/run_x11_vnc.sh
-        read_only: true
+  #     # force our noop script - socat isn't needed in host mode.
+  #     - type: bind
+  #       source: ./run_x11_vnc.sh
+  #       target: /root/scripts/run_x11_vnc.sh
+  #       read_only: true
 
-    # NOTE: to fill these out, define an `.env` file in the same dir as
-    # this compose file which looks something like:
-    environment:
-      TRADING_MODE: 'live'
-      VNC_SERVER_PASSWORD: 'doggy'
-      VNC_SERVER_PORT: '3004'
+  #   # NOTE: to fill these out, define an `.env` file in the same dir as
+  #   # this compose file which looks something like:
+  #   environment:
+  #     TRADING_MODE: 'live'
+  #     VNC_SERVER_PASSWORD: 'doggy'
+  #     VNC_SERVER_PORT: '3004'

piker/_daemon.py

@@ -35,12 +35,17 @@
 
 _root_dname = 'pikerd'
 
-_registry_host: str = '127.0.0.1'
-_registry_port: int = 6116
-_registry_addr = (
-    _registry_host,
-    _registry_port,
+_default_registry_host: str = '127.0.0.1'
+_default_registry_port: int = 6116
+_default_reg_addr: tuple[str, int] = (
+    _default_registry_host,
+    _default_registry_port,
 )
+
+# NOTE: this value is set as an actor-global once the first endpoint
+# who is capable, spawns a `pikerd` service tree.
+_registry_addr: tuple[str, int] | None = None
+
 _tractor_kwargs: dict[str, Any] = {
     # use a different registry addr then tractor's default
     'arbiter_addr': _registry_addr
@@ -152,13 +157,20 @@ async def open_pikerd(
 
     '''
     global _services
+    global _registry_addr
+
+    if (
+        _registry_addr is None
+        or registry_addr
+    ):
+        _registry_addr = registry_addr or _default_reg_addr
 
     # XXX: this may open a root actor as well
     async with (
         tractor.open_root_actor(
 
             # passed through to ``open_root_actor``
-            arbiter_addr=registry_addr or _registry_addr,
+            arbiter_addr=_registry_addr,
             name=_root_dname,
             loglevel=loglevel,
             debug_mode=debug_mode,
@@ -197,7 +209,7 @@ async def open_piker_runtime(
     # XXX: you should pretty much never want debug mode
     # for data daemons when running in production.
     debug_mode: bool = False,
-    registry_addr: None | tuple[str, int] = _registry_addr,
+    registry_addr: None | tuple[str, int] = None,
 
 ) -> tractor.Actor:
     '''
@@ -206,13 +218,20 @@ async def open_piker_runtime(
 
     '''
     global _services
+    global _registry_addr
+
+    if (
+        _registry_addr is None
+        or registry_addr
+    ):
+        _registry_addr = registry_addr or _default_reg_addr
 
     # XXX: this may open a root actor as well
     async with (
         tractor.open_root_actor(
 
             # passed through to ``open_root_actor``
-            arbiter_addr=registry_addr,
+            arbiter_addr=_registry_addr,
             name=name,
             loglevel=loglevel,
             debug_mode=debug_mode,

piker/brokers/__init__.py

@@ -26,10 +26,21 @@
 
 __brokers__ = [
     'binance',
-    'questrade',
-    'robinhood',
     'ib',
     'kraken',
+
+    # broken but used to work
+    # 'questrade',
+    # 'robinhood',
+
+    # TODO: we should get on these stat!
+    # alpaca
+    # wstrade
+    # iex
+
+    # deribit
+    # kucoin
+    # bitso
 ]
 
 

piker/brokers/binance.py

@@ -41,10 +41,15 @@
     SymbolNotFound,
     DataUnavailable,
 )
-from ..log import get_logger, get_console_log
-from ..data import ShmArray
+from ..log import (
+    get_logger,
+    get_console_log,
+)
 from ..data.types import Struct
-from ..data._web_bs import open_autorecon_ws, NoBsWs
+from ..data._web_bs import (
+    open_autorecon_ws,
+    NoBsWs,
+)
 
 log = get_logger(__name__)
 
@@ -142,7 +147,9 @@ class OHLC(Struct):
 
 
 # convert datetime obj timestamp to unixtime in milliseconds
-def binance_timestamp(when):
+def binance_timestamp(
+    when: datetime
+) -> int:
     return int((when.timestamp() * 1000) + (when.microsecond / 1000))
 
 
@@ -181,7 +188,7 @@ async def symbol_info(
         params = {}
 
         if sym is not None:
-            sym = sym.upper()
+            sym = sym.lower()
             params = {'symbol': sym}
 
         resp = await self._api(
@@ -238,7 +245,7 @@ async def bars(
     ) -> dict:
 
         if end_dt is None:
-            end_dt = pendulum.now('UTC')
+            end_dt = pendulum.now('UTC').add(minutes=1)
 
         if start_dt is None:
             start_dt = end_dt.start_of(
@@ -396,8 +403,8 @@ async def open_history_client(
 
         async def get_ohlc(
             timeframe: float,
-            end_dt: Optional[datetime] = None,
-            start_dt: Optional[datetime] = None,
+            end_dt: datetime | None = None,
+            start_dt: datetime | None = None,
 
         ) -> tuple[
             np.ndarray,
@@ -412,25 +419,20 @@ async def get_ohlc(
                 start_dt=start_dt,
                 end_dt=end_dt,
             )
-            start_dt = pendulum.from_timestamp(array[0]['time'])
-            end_dt = pendulum.from_timestamp(array[-1]['time'])
-            return array, start_dt, end_dt
+            times = array['time']
+            if (
+                end_dt is None
+            ):
+                inow = round(time.time())
+                if (inow - times[-1]) > 60:
+                    await tractor.breakpoint()
 
-        yield get_ohlc, {'erlangs': 3, 'rate': 3}
+            start_dt = pendulum.from_timestamp(times[0])
+            end_dt = pendulum.from_timestamp(times[-1])
 
+            return array, start_dt, end_dt
 
-async def backfill_bars(
-    sym: str,
-    shm: ShmArray,  # type: ignore # noqa
-    task_status: TaskStatus[trio.CancelScope] = trio.TASK_STATUS_IGNORED,
-) -> None:
-    """Fill historical bars into shared mem / storage afap.
-    """
-    with trio.CancelScope() as cs:
-        async with open_cached_client('binance') as client:
-            bars = await client.bars(symbol=sym)
-            shm.push(bars)
-            task_status.started(cs)
+        yield get_ohlc, {'erlangs': 3, 'rate': 3}
 
 
 async def stream_quotes(
@@ -465,7 +467,7 @@ async def stream_quotes(
             si = sym_infos[sym] = syminfo.to_dict()
             filters = {}
             for entry in syminfo.filters:
-                ftype = entry.pop('filterType')
+                ftype = entry['filterType']
                 filters[ftype] = entry
 
             # XXX: after manually inspecting the response format we

piker/brokers/ib/broker.py

@@ -371,8 +371,8 @@ async def update_and_audit_msgs(
                     else:
                         entry = f'split_ratio = 1/{int(reverse_split_ratio)}'
 
-                    raise ValueError(
-                    # log.error(
+                    # raise ValueError(
+                    log.error(
                         f'POSITION MISMATCH ib <-> piker ledger:\n'
                         f'ib: {ibppmsg}\n'
                         f'piker: {msg}\n'
@@ -575,17 +575,18 @@ async def trades_dialogue(
                         # if new trades are detected from the API, prepare
                         # them for the ledger file and update the pptable.
                         if api_to_ledger_entries:
-                            trade_entries = api_to_ledger_entries[acctid]
+                            trade_entries = api_to_ledger_entries.get(acctid)
 
-                            # write ledger with all new trades **AFTER**
-                            # we've updated the `pps.toml` from the
-                            # original ledger state! (i.e. this is
-                            # currently done on exit)
-                            ledger.update(trade_entries)
+                            if trade_entries:
+                                # write ledger with all new trades **AFTER**
+                                # we've updated the `pps.toml` from the
+                                # original ledger state! (i.e. this is
+                                # currently done on exit)
+                                ledger.update(trade_entries)
 
-                            trans = trans_by_acct.get(acctid)
-                            if trans:
-                                table.update_from_trans(trans)
+                                trans = trans_by_acct.get(acctid)
+                                if trans:
+                                    table.update_from_trans(trans)
 
                         # XXX: not sure exactly why it wouldn't be in
                         # the updated output (maybe this is a bug?) but
@@ -883,7 +884,7 @@ async def deliver_trade_events(
                     # execdict.pop('acctNumber')
 
                     fill_msg = BrokerdFill(
-                        # should match the value returned from
+                        # NOTE: should match the value returned from
                         # `.submit_limit()`
                         reqid=execu.orderId,
                         time_ns=time.time_ns(),  # cuz why not

piker/brokers/ib/feed.py

@@ -1047,7 +1047,13 @@ async def open_symbol_search(
                 stock_results = []
 
                 async def stash_results(target: Awaitable[list]):
-                    stock_results.extend(await target)
+                    try:
+                        results = await target
+                    except tractor.trionics.Lagged:
+                        print("IB SYM-SEARCH OVERRUN?!?")
+                        return
+
+                    stock_results.extend(results)
 
                 for i in range(10):
                     with trio.move_on_after(3) as cs: