Redesign core

.github/workflows/main.yml

@@ -2,9 +2,9 @@ name: CI
 
 on:
   push:
-    branches: [ main, update-** ]
+    branches: [main, update-**, redesign_core]
   pull_request:
-    branches: [ '**' ]
+    branches: ["**"]
 
 jobs:
   build:
@@ -13,7 +13,7 @@ jobs:
       matrix:
         python-version: ["3.10", "3.11", "3.12"]
         os: [macos-13, ubuntu-latest]
-        mpi: ["openmpi"]  # [ 'mpich', 'openmpi', 'intelmpi']
+        mpi: ["openmpi"] # [ 'mpich', 'openmpi', 'intelmpi']
         include:
           - os: macos-13
             path: ~/Library/Caches/pip
@@ -48,17 +48,14 @@ jobs:
           key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}
       - name: Install dependencies
         if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
-        run: poetry install --no-interaction --no-root --all-extras --with=dev,algorithmExtension,sortingExtension #,mpi
+        run: poetry install --no-interaction --all-extras --with=dev,algorithmExtension,sortingExtension #,mpi
       - uses: FedericoCarboni/setup-ffmpeg@v3
         id: setup-ffmpeg
         with:
           # like "6.1.0". At the moment semver specifiers (i.e. >=6.1.0) are supported
           # only on Windows, on other platforms they are allowed but version is matched
           # exactly regardless.
           ffmpeg-version: release
-          # Target architecture of the ffmpeg executable to install. Defaults to the
-          # system architecture. Only x64 and arm64 are supported (arm64 only on Linux).
-          architecture: ''
           # Linking type of the binaries. Use "shared" to download shared binaries and
           # "static" for statically linked ones. Shared builds are currently only available
           # for windows releases. Defaults to "static"
@@ -70,17 +67,17 @@ jobs:
       - name: Run pytests
         if: always()
         run: |
-          source $VENV
+          source .venv/bin/activate
           make test
       - name: Run mypy
         if: always()
         run: |
-          source $VENV
+          source .venv/bin/activate
           make mypy
       - name: Run formatting check
         if: always()
         run: |
-          source $VENV
+          source .venv/bin/activate
           make check-codestyle
       # Upload coverage to Codecov (use python 3.10 ubuntu-latest)
       - name: Upload coverage to Codecov (only on 3.10 ubuntu-latest)

.pre-commit-config.yaml

@@ -17,7 +17,7 @@ repos:
     hooks:
       - id: pyupgrade
         name: pyupgrade
-        entry: poetry run pyupgrade --py311-plus
+        entry: poetry run pyupgrade --py310-plus
         types: [python]
         language: system
 

Makefile

@@ -23,7 +23,7 @@ pre-commit-install:
 #* Formatters
 .PHONY: codestyle
 codestyle:
-	# poetry run pyupgrade --exit-zero-even-if-changed --py38-plus **/*.py
+	poetry run pyupgrade --exit-zero-even-if-changed --py310-plus **/*.py
 	# poetry run isort --settings-path pyproject.toml ./
 	poetry run black --config pyproject.toml ./
 
@@ -33,7 +33,8 @@ formatting: codestyle
 #* Linting
 .PHONY: test
 test:
-	poetry run pytest -c pyproject.toml --cov=miv --cov-report=xml
+	poetry run pytest -c pyproject.toml --cov=miv/core
+	# poetry run pytest -c pyproject.toml --cov=miv/core --cov-report=xml
 
 .PHONY: check-codestyle
 check-codestyle:
@@ -42,7 +43,7 @@ check-codestyle:
 
 .PHONY: mypy
 mypy:
-	poetry run mypy --config-file pyproject.toml miv
+	poetry run mypy --config-file pyproject.toml miv/core
 
 .PHONY: lint
 lint: test check-codestyle mypy check-safety

docs/Makefile

@@ -19,5 +19,8 @@ help:
 %: Makefile
 	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
 
+.PHONY: clean clean_cache
+clean:
+	rm -rf $(BUILDDIR)/*
 clean_cache:
 	rm -rf **/results **/datasets

docs/conf.py

@@ -85,7 +85,7 @@
 autosummary_generate = True
 autosummary_generate_overwrite = False
 
-source_parsers: Dict[str, str] = {}
+source_parsers: dict[str, str] = {}
 source_suffix = {
     ".rst": "restructuredtext",
     ".md": "myst-nb",

miv/core/datatype/__init__.py

@@ -2,10 +2,9 @@
 
 from miv.core.datatype.collapsable import *
 from miv.core.datatype.events import *
-from miv.core.datatype.protocol import *
-from miv.core.datatype.pure_python import *
-from miv.core.datatype.signal import *
-from miv.core.datatype.spikestamps import *
+from .pure_python import *
+from .signal import *
+from .spikestamps import *
 
 DataTypes = Any  # Union[ # TODO
 #    miv.core.datatype.signal.Signal,

miv/core/datatype/collapsable.py

@@ -1,20 +1,21 @@
-from typing import Generator, Protocol
-
-from miv.core.datatype.protocol import Extendable
+from typing import Any, Protocol
+from collections.abc import Iterable
 
 
 class _Collapsable(Protocol):
     @classmethod
-    def from_collapse(self) -> None: ...
+    def from_collapse(self, values: Iterable["_Collapsable"]) -> "_Collapsable": ...
 
+    def extend(self, *args: Any, **kwargs: Any) -> None: ...
 
-class CollapseExtendableMixin:
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
 
+class CollapseExtendableMixin:
     @classmethod
-    def from_collapse(cls, values: Generator[Extendable, None, None]):
-        obj = cls()
-        for value in values:
-            obj.extend(value)
+    def from_collapse(cls, values: Iterable[_Collapsable]) -> _Collapsable:
+        obj: _Collapsable
+        for idx, value in enumerate(values):
+            if idx == 0:
+                obj = value
+            else:
+                obj.extend(value)
         return obj

miv/core/datatype/events.py

@@ -9,7 +9,7 @@
 
 __all__ = ["Events"]
 
-from typing import List, Optional
+from typing import Optional, cast
 
 from collections import UserList
 
@@ -28,36 +28,36 @@ class Events(CollapseExtendableMixin, DataNodeMixin):
     Comply with `Extendable` protocols.
     """
 
-    def __init__(self, data: List[float] = None):
+    def __init__(self, data: list[float] | None = None) -> None:
         super().__init__()
-        self.data = np.asarray(data) if data is not None else []
+        self.data = np.asarray(data) if data is not None else np.array([])
 
-    def append(self, item):
-        raise NotImplementedError("Not implemented yet. Need to append and sort")
+    def append(self, item: float) -> None:
+        self.data = np.append(self.data, item)
 
-    def extend(self, other):
-        raise NotImplementedError("Not implemented yet. Need to extend and sort")
+    def extend(self, other: "Events") -> None:
+        self.data = np.append(self.data, other.data)
 
-    def __len__(self):
+    def __len__(self) -> int:
         return len(self.data)
 
-    def get_last_event(self):
+    def get_last_event(self) -> float:
         """Return timestamps of the last event"""
-        return max(self.data)
+        return cast(float, max(self.data))
 
-    def get_first_event(self):
+    def get_first_event(self) -> float:
         """Return timestamps of the first event"""
-        return min(self.data)
+        return cast(float, min(self.data))
 
-    def get_view(self, t_start: float, t_end: float):
+    def get_view(self, t_start: float, t_end: float) -> "Events":
         """Truncate array and only includes spikestamps between t_start and t_end."""
         return Events(sorted(list(filter(lambda x: t_start <= x <= t_end, self.data))))
 
     def binning(
         self,
-        bin_size: float = 1 * pq.ms,
-        t_start: Optional[float] = None,
-        t_end: Optional[float] = None,
+        bin_size: float | pq.Quantity = 0.001,
+        t_start: float | None = None,
+        t_end: float | None = None,
         return_count: bool = False,
     ) -> Signal:
         """
@@ -89,6 +89,7 @@ def binning(
             rate=1.0 / bin_size,
         )
 
+        # TODO: Make separate free function for this binning process
         bins = np.digitize(self.data, time)
         bincount = np.bincount(bins, minlength=n_bins + 2)[1:-1]
         if return_count:

miv/core/datatype/pure_python.py

@@ -1,31 +1,26 @@
 __all__ = ["PythonDataType", "NumpyDType", "GeneratorType"]
 
-from typing import Protocol, Union
+from typing import Protocol, Union, TypeAlias, Any
+from collections.abc import Generator, Iterator
 
 import numpy as np
 
+from miv.core.operator.operator import DataNodeMixin
 from miv.core.operator.chainable import BaseChainingMixin
 
-
-class RawValuesProtocol(Protocol):
-    @staticmethod
-    def is_valid(value) -> bool: ...
+PurePythonTypes: TypeAlias = Union[int, float, str, bool, list, tuple, dict]
 
 
-class ValuesMixin(BaseChainingMixin):
+class ValuesMixin(DataNodeMixin, BaseChainingMixin):
     """
     This mixin is used to convert pure/numpy data type to be a valid input/output of a node.
     """
 
-    def __init__(self, value, *args, **kwargs):
+    def __init__(
+        self, data: np.ndarray | PurePythonTypes, *args: Any, **kwargs: Any
+    ) -> None:
         super().__init__(*args, **kwargs)
-        self.value = value
-
-    def output(self):
-        return self.value
-
-    def run(self, *args, **kwargs):
-        return self.output()
+        self.data = data
 
 
 class PythonDataType(ValuesMixin):
@@ -35,9 +30,9 @@ class PythonDataType(ValuesMixin):
     """
 
     @staticmethod
-    def is_valid(value):
-        return value is None or isinstance(
-            value, (int, float, str, bool, list, tuple, dict)
+    def is_valid(data: Any) -> bool:
+        return data is None or isinstance(
+            data, (int, float, str, bool, list, tuple, dict)
         )
 
 
@@ -47,23 +42,23 @@ class NumpyDType(ValuesMixin):
     """
 
     @staticmethod
-    def is_valid(value):
-        return isinstance(value, np.ndarray)
+    def is_valid(data: Any) -> bool:
+        return isinstance(data, np.ndarray)
 
 
 class GeneratorType(BaseChainingMixin):
-    def __init__(self, iterator, *args, **kwargs):
+    def __init__(self, iterator: Iterator, *args: Any, **kwargs: Any) -> None:
         super().__init__(*args, **kwargs)
         self.iterator = iterator
 
-    def output(self):
+    def output(self) -> Generator:
         yield from self.iterator
 
-    def run(self, **kwargs):
+    def run(self, **kwargs: Any) -> Generator:
         yield from self.output()
 
     @staticmethod
-    def is_valid(value):
+    def is_valid(data: Any) -> bool:
         import inspect
 
-        return inspect.isgenerator(value)
+        return inspect.isgenerator(data)