diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index c2a0ca2..a658a76 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -117,14 +117,19 @@ jobs: platform: - runner: windows-latest target: x64 + python_arch: x64 - runner: windows-latest target: x86 + python_arch: x86 + - runner: windows-11-arm + target: aarch64 + python_arch: arm64 steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: 3.x - architecture: ${{ matrix.platform.target }} + architecture: ${{ matrix.platform.python_arch }} - name: Build wheels uses: PyO3/maturin-action@db323e2cf5679b7feb8bcb561a36b27a0bc19e79 # v1 with: diff --git a/src/tiktoken.rs b/src/tiktoken.rs index 9c5a290..f4eb1bc 100644 --- a/src/tiktoken.rs +++ b/src/tiktoken.rs @@ -305,12 +305,7 @@ impl CoreBPE { let token_is_all_space = |token| { self.decoder .get(token) - .map(|token_bytes| { - token_bytes - .iter() - .rev() - .all(|&b| [b' ', b'\n', b'\t'].contains(&b)) - }) + .map(|token_bytes| token_bytes.iter().rev().all(|&b| b" \n\t".contains(&b))) .unwrap_or(false) }; if last_piece_token_len > 0