From 61497f59c6b0b1809b2fc8a3e73f05a63bea1b38 Mon Sep 17 00:00:00 2001 From: DarkSharpness Date: Mon, 24 Feb 2025 08:16:44 +0000 Subject: [PATCH 01/12] chore(ci): try to add unit_test ci --- .github/workflows/unit_test.yaml | 35 ++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 .github/workflows/unit_test.yaml diff --git a/.github/workflows/unit_test.yaml b/.github/workflows/unit_test.yaml new file mode 100644 index 00000000..5e59eace --- /dev/null +++ b/.github/workflows/unit_test.yaml @@ -0,0 +1,35 @@ +on: [push] + +jobs: + run_unit_test: + name: Run unit test + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Python 3.11 + uses: actions/setup-python@v5 + with: + python-version: 3.11 + + - name: Install dependencies + run: | + sudo apt install -y gcc g++ cmake + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install pybind11 ninja + + - name: Build xgrammar and run C++ tests + run: | + echo "\nset(XGRAMMAR_BUILD_CXX_TESTS ON)\n" >> cmake/config.cmake + mkdir build && cd build + cmake .. -G Ninja + ninja && ctest + cd .. + + - name: Run Python tests + run: | + cd python && pip install -e . && cd .. + python -m pytest tests/python + timeout-minutes: 5 # to avoid blocking the CI From 0e3aca3b9b9bc868401e86cc23b9446d997ee0ef Mon Sep 17 00:00:00 2001 From: DarkSharpness Date: Mon, 24 Feb 2025 08:20:26 +0000 Subject: [PATCH 02/12] chore(ci): minor fix --- .github/workflows/unit_test.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/unit_test.yaml b/.github/workflows/unit_test.yaml index 5e59eace..09a5c6cc 100644 --- a/.github/workflows/unit_test.yaml +++ b/.github/workflows/unit_test.yaml @@ -17,8 +17,7 @@ jobs: run: | sudo apt install -y gcc g++ cmake python -m pip install --upgrade pip - pip install -r requirements.txt - pip install pybind11 ninja + python -m pip install ninja pybind11 torch - name: Build xgrammar and run C++ tests run: | From 671aa790cad5ce53693f11a4865cae7b125fabae Mon Sep 17 00:00:00 2001 From: DarkSharpness Date: Mon, 24 Feb 2025 08:24:20 +0000 Subject: [PATCH 03/12] chore(ci): minor fix --- .github/workflows/unit_test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit_test.yaml b/.github/workflows/unit_test.yaml index 09a5c6cc..ff1c84ec 100644 --- a/.github/workflows/unit_test.yaml +++ b/.github/workflows/unit_test.yaml @@ -21,7 +21,7 @@ jobs: - name: Build xgrammar and run C++ tests run: | - echo "\nset(XGRAMMAR_BUILD_CXX_TESTS ON)\n" >> cmake/config.cmake + echo "set(XGRAMMAR_BUILD_CXX_TESTS ON)" >> cmake/config.cmake mkdir build && cd build cmake .. -G Ninja ninja && ctest From ade3bee6374e7fd9406618985e21949ebfe7f86e Mon Sep 17 00:00:00 2001 From: DarkSharpness Date: Mon, 24 Feb 2025 08:29:41 +0000 Subject: [PATCH 04/12] chore(ci): minor fix --- .github/workflows/unit_test.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/unit_test.yaml b/.github/workflows/unit_test.yaml index ff1c84ec..7f3a110e 100644 --- a/.github/workflows/unit_test.yaml +++ b/.github/workflows/unit_test.yaml @@ -8,6 +8,9 @@ jobs: - name: Checkout uses: actions/checkout@v4 + - name: Init submodules + run: git submodule update --init --recursive + - name: Set up Python 3.11 uses: actions/setup-python@v5 with: From ff81f14aedb0c7a76ffa62b6dcf69d2b99be7912 Mon Sep 17 00:00:00 2001 From: DarkSharpness Date: Tue, 25 Feb 2025 05:30:52 +0000 Subject: [PATCH 05/12] chore(ci): support pytest without hf token --- .github/workflows/unit_test.yaml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/unit_test.yaml b/.github/workflows/unit_test.yaml index 7f3a110e..b3add912 100644 --- a/.github/workflows/unit_test.yaml +++ b/.github/workflows/unit_test.yaml @@ -32,6 +32,5 @@ jobs: - name: Run Python tests run: | - cd python && pip install -e . && cd .. - python -m pytest tests/python - timeout-minutes: 5 # to avoid blocking the CI + pip install -e . + pytest -m "not hf_token_required" From ced01389a1fd6163a7a3a65fcb3ffc72c97ed373 Mon Sep 17 00:00:00 2001 From: DarkSharpness Date: Tue, 25 Feb 2025 05:38:06 +0000 Subject: [PATCH 06/12] chore(ci): simplify ci --- .github/workflows/unit_test.yaml | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/.github/workflows/unit_test.yaml b/.github/workflows/unit_test.yaml index b3add912..e570ee50 100644 --- a/.github/workflows/unit_test.yaml +++ b/.github/workflows/unit_test.yaml @@ -16,21 +16,15 @@ jobs: with: python-version: 3.11 - - name: Install dependencies + - name: Build xgrammar from source run: | - sudo apt install -y gcc g++ cmake - python -m pip install --upgrade pip - python -m pip install ninja pybind11 torch + echo "set(XGRAMMAR_BUILD_CXX_TESTS ON)" >> cmake/config.cmake + pip install -e . - - name: Build xgrammar and run C++ tests + - name: Run C++ tests run: | - echo "set(XGRAMMAR_BUILD_CXX_TESTS ON)" >> cmake/config.cmake - mkdir build && cd build - cmake .. -G Ninja - ninja && ctest - cd .. + cd build && ctest && cd .. - name: Run Python tests run: | - pip install -e . pytest -m "not hf_token_required" From 733e6edebd156f43b9cf98a72ce75cc76c394fae Mon Sep 17 00:00:00 2001 From: DarkSharpness Date: Tue, 25 Feb 2025 06:28:33 +0000 Subject: [PATCH 07/12] chore(ci): minor fix --- .github/workflows/unit_test.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/unit_test.yaml b/.github/workflows/unit_test.yaml index e570ee50..10199966 100644 --- a/.github/workflows/unit_test.yaml +++ b/.github/workflows/unit_test.yaml @@ -27,4 +27,5 @@ jobs: - name: Run Python tests run: | + pip install pytest pytest -m "not hf_token_required" From 3467f57ca8514742e18ec676f23b9f6d8583e279 Mon Sep 17 00:00:00 2001 From: DarkSharpness Date: Tue, 25 Feb 2025 06:57:38 +0000 Subject: [PATCH 08/12] chore(ci): try to support multi-arch --- .github/workflows/unit_test.yaml | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/.github/workflows/unit_test.yaml b/.github/workflows/unit_test.yaml index 10199966..1fc2bc3b 100644 --- a/.github/workflows/unit_test.yaml +++ b/.github/workflows/unit_test.yaml @@ -1,15 +1,27 @@ -on: [push] +on: + workflow_dispatch: + pull_request: + push: jobs: run_unit_test: - name: Run unit test - runs-on: ubuntu-latest + name: Run unit tests on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: + - {os: ubuntu-latest, arch: x86_64} + - {os: ubuntu-24.04-arm, arch: aarch64} + - {os: windows-latest, arch: AMD64} + - {os: macos-14, arch: arm64} + - {os: macos-13, arch: x86_64,} + steps: - name: Checkout uses: actions/checkout@v4 - - - name: Init submodules - run: git submodule update --init --recursive + with: + submodules: recursive - name: Set up Python 3.11 uses: actions/setup-python@v5 @@ -19,6 +31,7 @@ jobs: - name: Build xgrammar from source run: | echo "set(XGRAMMAR_BUILD_CXX_TESTS ON)" >> cmake/config.cmake + python -m pip install --upgrade pip pip install -e . - name: Run C++ tests @@ -26,6 +39,9 @@ jobs: cd build && ctest && cd .. - name: Run Python tests + if: ${{ secrets.HF_TOKEN != '' }} + env: + HF_TOKEN: ${{ secrets.HF_TOKEN }} run: | pip install pytest - pytest -m "not hf_token_required" + pytest From 39df3bcc40b2fdb28499ba0ae72bd957cbb8233c Mon Sep 17 00:00:00 2001 From: DarkSharpness Date: Tue, 25 Feb 2025 07:02:31 +0000 Subject: [PATCH 09/12] chore(ci): try fix --- .github/workflows/unit_test.yaml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/unit_test.yaml b/.github/workflows/unit_test.yaml index 1fc2bc3b..a78ecb72 100644 --- a/.github/workflows/unit_test.yaml +++ b/.github/workflows/unit_test.yaml @@ -39,9 +39,17 @@ jobs: cd build && ctest && cd .. - name: Run Python tests - if: ${{ secrets.HF_TOKEN != '' }} env: HF_TOKEN: ${{ secrets.HF_TOKEN }} + if: env.HF_TOKEN != '' run: | pip install pytest pytest + + - name: Run Python tests without HF_TOKEN + env: + HF_TOKEN: ${{ secrets.HF_TOKEN }} + if: env.HF_TOKEN == '' + run: | + pip install pytest + pytest -m "not hf_token_required" From 3911674ab8d0ae95c5e1689fa6e2328454ca7655 Mon Sep 17 00:00:00 2001 From: DarkSharpness Date: Tue, 25 Feb 2025 07:08:57 +0000 Subject: [PATCH 10/12] fix: fix use of deprecated std::result_of --- cpp/support/thread_pool.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cpp/support/thread_pool.h b/cpp/support/thread_pool.h index b48fc5ca..0146f571 100644 --- a/cpp/support/thread_pool.h +++ b/cpp/support/thread_pool.h @@ -12,6 +12,7 @@ #include #include #include +#include #include #include "logging.h" @@ -67,9 +68,8 @@ class ThreadPool { * \note Tasks are executed in FIFO order but may complete in any order. */ template - auto Submit(F&& f, Args&&... args) - -> std::shared_future::type> { - using return_type = typename std::result_of::type; + auto Submit(F&& f, Args&&... args) -> std::shared_future> { + using return_type = std::invoke_result_t; // Package the task with its arguments into a shared pointer auto task = std::make_shared>( From 3d6fc8c3044461e12eec4e647c8b3a891c8fae25 Mon Sep 17 00:00:00 2001 From: DarkSharpness Date: Tue, 25 Feb 2025 07:25:06 +0000 Subject: [PATCH 11/12] fix: fix use of designated initializers --- tests/cpp/test_thread_safe_cache.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/cpp/test_thread_safe_cache.cc b/tests/cpp/test_thread_safe_cache.cc index 8413358f..bfefda5b 100644 --- a/tests/cpp/test_thread_safe_cache.cc +++ b/tests/cpp/test_thread_safe_cache.cc @@ -34,7 +34,9 @@ using namespace std::chrono_literals; TEST(XGrammarParallelTest, CacheEfficiency) { auto cache = ThreadSafeCache{[](const std::string&) { std::this_thread::sleep_for(1s); // simulate a slow operation - return MockGrammar{.uuid = counter++, .padding = {}}; + MockGrammar g{}; + g.uuid = counter++; + return g; }}; auto futures = std::vector>{}; From 5874f80c0eb770fe1877576735d7f8f7b91af9b5 Mon Sep 17 00:00:00 2001 From: DarkSharpness Date: Wed, 26 Feb 2025 02:41:18 +0000 Subject: [PATCH 12/12] chore(ci): format and simplify --- .github/workflows/unit_test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit_test.yaml b/.github/workflows/unit_test.yaml index a78ecb72..86e43099 100644 --- a/.github/workflows/unit_test.yaml +++ b/.github/workflows/unit_test.yaml @@ -36,7 +36,7 @@ jobs: - name: Run C++ tests run: | - cd build && ctest && cd .. + ctest --test-dir build - name: Run Python tests env: