Merge pull request #191 from promised-ai/feature/pitman-yor

Allow users to specify prior process for row and column partitions
promised-ai · Jun 25, 2024 · d09a665 · d09a665
2 parents f57a371 + 0d405de
commit d09a665
Show file tree

Hide file tree

Showing 140 changed files with 4,891 additions and 2,785 deletions.
diff --git a/.github/scripts/find_compatible_wheel.py b/.github/scripts/find_compatible_wheel.py
@@ -10,23 +10,29 @@
     description="Program to find wheels in a directory compatible with the current version of Python"
 )
 
-parser.add_argument("package", help="The name of the package that you are searching for a wheel for")
+parser.add_argument(
+    "package", help="The name of the package that you are searching for a wheel for"
+)
 parser.add_argument("dir", help="the directory under which to search for the wheels")
 
-args=parser.parse_args()
+args = parser.parse_args()
 
-wheel=None
+wheel = None
 
 for tag in sys_tags():
     print(f"Looking for file matching tag {tag}", file=sys.stderr)
-    matches=glob.glob(args.package + "*" + str(tag) + "*.whl", root_dir=args.dir)
+    matches = glob.glob(f"{args.package}*{tag}*.whl", root_dir=args.dir)
     if len(matches) == 1:
-        wheel=matches[0]
+        wheel = matches[0]
         break
     elif len(matches) > 1:
-        print("Found multiple matches for the same tag " + str(tag), matches, file=sys.stderr)
+        print(
+            f"Found multiple matches for the same tag `{tag}`",
+            matches,
+            file=sys.stderr,
+        )
 
-if wheel: 
+if wheel:
     print(os.path.join(args.dir, wheel))
 else:
     sys.exit("Did not find compatible wheel")
diff --git a/.github/workflows/changelog.yaml b/.github/workflows/changelog.yaml
@@ -12,7 +12,7 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
-      
+
       - run: |
           eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)"
           brew install nbbrd/tap/heylogs

diff --git a/.github/workflows/python-build-test.yaml b/.github/workflows/python-build-test.yaml
@@ -20,7 +20,7 @@ jobs:
     defaults:
       run:
         working-directory: pylace
-    
+
     steps:
       - uses: actions/checkout@v4
 
@@ -30,7 +30,7 @@ jobs:
           python-version: '3.12'
           cache: 'pip'
           cache-dependency-path: "pylace/requirements-lint.txt"
-      
+
       - name: Install Python dependencies
         run: |
           pip install --upgrade pip
@@ -102,18 +102,18 @@ jobs:
       - name: Build wheels
         uses: PyO3/maturin-action@v1
         with:
+          maturin-version: 1.5.1
           target: ${{ matrix.target }}
           args: --release --out dist -i python3.8 -i python3.9 -i python3.10 -i python3.11 -i python3.12 --manifest-path pylace/Cargo.toml
           manylinux: auto
-    
+
       - name: Install dev dependencies
         run: |
           pip install --upgrade pip
           pip install -r pylace/requirements-dev.txt
 
       - name: Install pylace
         run: |
-          ls -l ./dist
           WHEEL_FILE=$(python3 .github/scripts/find_compatible_wheel.py pylace ./dist)
           echo "Installing $WHEEL_FILE"
           pip install $WHEEL_FILE
@@ -147,17 +147,17 @@ jobs:
       - name: Build wheels
         uses: PyO3/maturin-action@v1
         with:
+          maturin-version: 1.5.1
           target: ${{ matrix.target }}
           args: --release --out dist -i python3.8 -i python3.9 -i python3.10 -i python3.11 -i python3.12 --manifest-path pylace/Cargo.toml
-    
+
       - name: Install dev dependencies
         run: |
           pip install --upgrade pip
           pip install -r pylace/requirements-dev.txt
 
       - name: Install pylace
         run: |
-          ls -l ./dist
           $WHEEL_FILE = (python3 .github/scripts/find_compatible_wheel.py pylace ./dist)
           echo "Installing $WHEEL_FILE"
           pip install $WHEEL_FILE
@@ -172,11 +172,15 @@ jobs:
           path: dist
 
   macos:
-    runs-on: macos-latest
     needs: ["lint-python", "lint-rust"]
     strategy:
       matrix:
-        target: [x86_64, aarch64]
+        include:
+          - os: macos-latest
+            target: aarch64
+          - os: macos-13
+            target: x86_64
+    runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
@@ -190,24 +194,22 @@ jobs:
       - name: Build wheels
         uses: PyO3/maturin-action@v1
         with:
+          maturin-version: 1.5.1
           target: ${{ matrix.target }}
           args: --release --out dist -i python3.8 -i python3.9 -i python3.10 -i python3.11 -i python3.12 --manifest-path pylace/Cargo.toml
 
       - name: Install dev dependencies
-        if: ${{ matrix.target != 'aarch64' }}
         run: |
           pip install --upgrade pip
           pip install -r pylace/requirements-dev.txt
 
       - name: Install pylace
-        if: ${{ matrix.target != 'aarch64' }}
         run: |
           WHEEL_FILE=$(python3 .github/scripts/find_compatible_wheel.py pylace ./dist)
           echo "Installing $WHEEL_FILE"
           pip install $WHEEL_FILE
 
       - name: Run Tests
-        if: ${{ matrix.target != 'aarch64' }}
         run: pytest pylace/tests
 
       - name: Upload wheels

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -5,7 +5,33 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## [Unreleased]
+## [rust-0.8.0] - 2024-04-10
+
+### Added
+
+- Added ability Pitman-Yor prior process
+    - Users now specify the prior process (and its prior) in the codebook
+- `lace_stats::prior_process::Builder`
+
+### Changed
+
+- Offloaded a lot of `Assignment` functionality to `PriorProcess`
+- `StateAlpha` and `ViewAlpha` transitions are now `StatePriorProcessParams` and `ViewPriorProcessParams`
+- Changed `SerializedType` default to `Bincode`
+- moved `Assignment` from `lace_cc` to `lace_stats`
+
+## [python-0.8.0] - 2024-04-10
+
+### Added
+
+- Added ability Pitman-Yor prior process.
+- Added `remove_rows` to `Engine`.
+- Added `with_index` to `CodeBook`.
+
+### Changed
+
+- `StateAlpha` and `ViewAlpha` transitions are now `StatePriorProcessParams` and `ViewPriorProcessParams`
+- Updated Pyo3 version to 0.21
 
 ## [python-0.7.1] - 2024-02-27
 
@@ -278,7 +304,9 @@ Initial release on [PyPi](https://pypi.org/)
 
 Initial release on [crates.io](https://crates.io/)
 
-[unreleased]: https://github.com/promised-ai/lace/compare/python-0.7.1...HEAD
+[unreleased]: https://github.com/promised-ai/lace/compare/python-0.8.0...HEAD
+[rust-0.8.0]: https://github.com/promised-ai/lace/compare/rust-0.7.0...rust-0.8.0
+[python-0.8.0]: https://github.com/promised-ai/lace/compare/python-0.7.1...python-0.8.0
 [python-0.7.1]: https://github.com/promised-ai/lace/compare/python-0.7.0...python-0.7.1
 [python-0.7.0]: https://github.com/promised-ai/lace/compare/python-0.6.0...python-0.7.0
 [rust-0.7.0]: https://github.com/promised-ai/lace/compare/rust-0.6.0...rust-0.7.0

diff --git a/CITATION.cff b/CITATION.cff
@@ -34,5 +34,5 @@ keywords:
   - Bayesian
   - Machine Learning
 license: BUSL-1.1
-version: 0.7.0
+version: 0.8.0
 date-released: '2024-02-07'
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -1,5 +1,8 @@
 # Contributing to Lace
 
+> :warning: Note that we are in the process of a major refector and are unlikely
+> to accept any contributions apart from simple bugfixes
+
 ## General Guidelines
 
 - Don't use getters and setters if it causes indirection in a performance heavy

diff --git a/book/lace_preprocess_mdbook_yaml/Cargo.lock b/book/lace_preprocess_mdbook_yaml/Cargo.lock
diff --git a/book/lace_preprocess_mdbook_yaml/Cargo.toml b/book/lace_preprocess_mdbook_yaml/Cargo.toml
@@ -16,8 +16,8 @@ path = "src/main.rs"
 anyhow = "1.0"
 clap = "4.2"
 env_logger = "0.10"
-lace_codebook = { path = "../../lace/lace_codebook", version = "0.6.0" }
-lace_stats = { path = "../../lace/lace_stats", version = "0.3.0" }
+lace_codebook = { path = "../../lace/lace_codebook", version = "0.7.0" }
+lace_stats = { path = "../../lace/lace_stats", version = "0.4.0" }
 log = "0.4"
 mdbook = "0.4"
 pulldown-cmark = { version = "0.9", default-features = false }

diff --git a/book/lace_preprocess_mdbook_yaml/src/lib.rs b/book/lace_preprocess_mdbook_yaml/src/lib.rs
@@ -1,5 +1,3 @@
-use std::collections::HashMap;
-
 use anyhow::anyhow;
 use log::debug;
 use mdbook::{
@@ -12,7 +10,15 @@ use regex::Regex;
 
 use serde::Deserialize;
 
-type GammaMap = HashMap<String, lace_stats::rv::dist::Gamma>;
+#[derive(Deserialize)]
+struct ViewPriorProcess {
+    pub view_prior_process: lace_codebook::PriorProcess,
+}
+
+#[derive(Deserialize)]
+struct StatePriorProcess {
+    pub state_prior_process: lace_codebook::PriorProcess,
+}
 
 fn check_deserialize_yaml<T>(input: &str) -> anyhow::Result<()>
 where
@@ -54,17 +60,14 @@ macro_rules! check_deserialize_arm {
     }
 }
 
-fn check_deserialize_dyn(
-    input: &str,
-    type_name: &str,
-    format: &str,
-) -> anyhow::Result<()> {
+fn check_deserialize_dyn(input: &str, type_name: &str, format: &str) -> anyhow::Result<()> {
     check_deserialize_arm!(
         input,
         type_name,
         format,
         [
-            GammaMap,
+            ViewPriorProcess,
+            StatePriorProcess,
             lace_codebook::ColMetadata,
             lace_codebook::ColMetadataList
         ]
@@ -80,35 +83,21 @@ impl YamlTester {
         YamlTester
     }
 
-    fn examine_chapter_content(
-        &self,
-        content: &str,
-        re: &Regex,
-    ) -> anyhow::Result<()> {
+    fn examine_chapter_content(&self, content: &str, re: &Regex) -> anyhow::Result<()> {
         let parser = Parser::new(content);
         let mut code_block = Some(String::new());
 
         for event in parser {
             match event {
-                Event::Start(Tag::CodeBlock(CodeBlockKind::Fenced(
-                    ref code_block_string,
-                ))) => {
+                Event::Start(Tag::CodeBlock(CodeBlockKind::Fenced(ref code_block_string))) => {
                     if re.is_match(code_block_string) {
-                        debug!(
-                            "YAML Block Start, identifier string={}",
-                            code_block_string
-                        );
+                        debug!("YAML Block Start, identifier string={}", code_block_string);
                         code_block = Some(String::new());
                     }
                 }
-                Event::End(Tag::CodeBlock(CodeBlockKind::Fenced(
-                    ref code_block_string,
-                ))) => {
+                Event::End(Tag::CodeBlock(CodeBlockKind::Fenced(ref code_block_string))) => {
                     if let Some(captures) = re.captures(code_block_string) {
-                        debug!(
-                            "Code Block End, identifier string={}",
-                            code_block_string
-                        );
+                        debug!("Code Block End, identifier string={}", code_block_string);
 
                         let serialization_format = captures
                             .get(1)
@@ -119,21 +108,13 @@ impl YamlTester {
                             .get(2)
                             .ok_or(anyhow!("No deserialize type found"))?
                             .as_str();
-                        debug!(
-                            "Target deserialization type is {}",
-                            target_type
-                        );
+                        debug!("Target deserialization type is {}", target_type);
 
                         let final_block = code_block.take();
-                        let final_block =
-                            final_block.ok_or(anyhow!("No YAML text found"))?;
+                        let final_block = final_block.ok_or(anyhow!("No YAML text found"))?;
                         debug!("Code block ended up as\n{}", final_block);
 
-                        check_deserialize_dyn(
-                            &final_block,
-                            target_type,
-                            serialization_format,
-                        )?;
+                        check_deserialize_dyn(&final_block, target_type, serialization_format)?;
                     }
                 }
                 Event::Text(ref text) => {
@@ -154,11 +135,7 @@ impl Preprocessor for YamlTester {
         "lace-yaml-tester"
     }
 
-    fn run(
-        &self,
-        _ctx: &PreprocessorContext,
-        book: Book,
-    ) -> anyhow::Result<Book> {
+    fn run(&self, _ctx: &PreprocessorContext, book: Book) -> anyhow::Result<Book> {
         debug!("Starting the run");
         let re = Regex::new(r"^(yaml|json).*,deserializeTo=([^,]+)").unwrap();
         for book_item in book.iter() {

diff --git a/book/src/SUMMARY.md b/book/src/SUMMARY.md
@@ -16,6 +16,7 @@
     - [Data Simulation](./pcc/simulate.md)
     - [In- and out-of-table operations](./pcc/inouttable.md)
     - [Adding data to a model](./pcc/add-data.md)
+    - [Prior processes](./pcc/prior-processes.md)
 - [Preparing your data](./data/basics.md)
 - [Codebook reference](./codebook-ref.md)
 - [Appendix](./appendix/appendix.md)
-Original file line number
+Diff line change
@@ Expand Up / @@ -12,7 +12,7 @@ jobs: @@
         steps:
           - uses: actions/checkout@v4
           - run: |
               eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)"
               brew install nbbrd/tap/heylogs
@@ Expand Down @@