Add CI support for Windows ARM64 Python wheels

.github/workflows/CI.yml

@@ -117,14 +117,19 @@ jobs:
         platform:
           - runner: windows-latest
             target: x64
+            python_arch: x64
           - runner: windows-latest
             target: x86
+            python_arch: x86
+          - runner: windows-11-arm
+            target: aarch64
+            python_arch: arm64
     steps:
       - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
       - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
         with:
           python-version: 3.x
-          architecture: ${{ matrix.platform.target }}
+          architecture: ${{ matrix.platform.python_arch }}
       - name: Build wheels
         uses: PyO3/maturin-action@db323e2cf5679b7feb8bcb561a36b27a0bc19e79 # v1
         with:

src/tiktoken.rs

@@ -305,12 +305,7 @@ impl CoreBPE {
             let token_is_all_space = |token| {
                 self.decoder
                     .get(token)
-                    .map(|token_bytes| {
-                        token_bytes
-                            .iter()
-                            .rev()
-                            .all(|&b| [b' ', b'\n', b'\t'].contains(&b))
-                    })
+                    .map(|token_bytes| token_bytes.iter().rev().all(|&b| b" \n\t".contains(&b)))
                     .unwrap_or(false)
             };
             if last_piece_token_len > 0