Skip to content

Commit f4c9fd7

Browse files
NarsilNarsil
authored
Testing ABI3 wheels to reduce number of wheels (#1674)
* Testing ABI3 wheels to reduce number of wheels * No need for py-clone anymore. * Upgrade python versions. * Remove those flakes. * Promoting new CI + Fixing secret.
1 parent 5aa9f6c commit f4c9fd7

File tree

14 files changed

+246
-66
lines changed

14 files changed

+246
-66
lines changed

‎.github/workflows/CI.yml

Lines changed: 181 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,181 @@
1+
# This file is autogenerated by maturin v1.7.4
2+
# To update, run
3+
#
4+
# maturin generate-ci github -m bindings/python/Cargo.toml
5+
#
6+
name: CI
7+
8+
on:
9+
push:
10+
branches:
11+
- main
12+
- master
13+
tags:
14+
- '*'
15+
pull_request:
16+
workflow_dispatch:
17+
18+
permissions:
19+
contents: read
20+
21+
jobs:
22+
linux:
23+
runs-on: ${{ matrix.platform.runner }}
24+
strategy:
25+
matrix:
26+
platform:
27+
- runner: ubuntu-latest
28+
target: x86_64
29+
- runner: ubuntu-latest
30+
target: x86
31+
- runner: ubuntu-latest
32+
target: aarch64
33+
- runner: ubuntu-latest
34+
target: armv7
35+
- runner: ubuntu-latest
36+
target: s390x
37+
- runner: ubuntu-latest
38+
target: ppc64le
39+
steps:
40+
- uses: actions/checkout@v4
41+
- uses: actions/setup-python@v5
42+
with:
43+
python-version: 3.x
44+
- name: Build wheels
45+
uses: PyO3/maturin-action@v1
46+
with:
47+
target: ${{ matrix.platform.target }}
48+
args: --release --out dist --manifest-path bindings/python/Cargo.toml
49+
sccache: 'true'
50+
manylinux: auto
51+
- name: Upload wheels
52+
uses: actions/upload-artifact@v4
53+
with:
54+
name: wheels-linux-${{ matrix.platform.target }}
55+
path: dist
56+
57+
musllinux:
58+
runs-on: ${{ matrix.platform.runner }}
59+
strategy:
60+
matrix:
61+
platform:
62+
- runner: ubuntu-latest
63+
target: x86_64
64+
- runner: ubuntu-latest
65+
target: x86
66+
- runner: ubuntu-latest
67+
target: aarch64
68+
- runner: ubuntu-latest
69+
target: armv7
70+
steps:
71+
- uses: actions/checkout@v4
72+
- uses: actions/setup-python@v5
73+
with:
74+
python-version: 3.x
75+
- name: Build wheels
76+
uses: PyO3/maturin-action@v1
77+
with:
78+
target: ${{ matrix.platform.target }}
79+
args: --release --out dist --manifest-path bindings/python/Cargo.toml
80+
sccache: 'true'
81+
manylinux: musllinux_1_2
82+
- name: Upload wheels
83+
uses: actions/upload-artifact@v4
84+
with:
85+
name: wheels-musllinux-${{ matrix.platform.target }}
86+
path: dist
87+
88+
windows:
89+
runs-on: ${{ matrix.platform.runner }}
90+
strategy:
91+
matrix:
92+
platform:
93+
- runner: windows-latest
94+
target: x64
95+
- runner: windows-latest
96+
target: x86
97+
steps:
98+
- uses: actions/checkout@v4
99+
- uses: actions/setup-python@v5
100+
with:
101+
python-version: 3.x
102+
architecture: ${{ matrix.platform.target }}
103+
- name: Build wheels
104+
uses: PyO3/maturin-action@v1
105+
with:
106+
target: ${{ matrix.platform.target }}
107+
args: --release --out dist --manifest-path bindings/python/Cargo.toml
108+
sccache: 'true'
109+
- name: Upload wheels
110+
uses: actions/upload-artifact@v4
111+
with:
112+
name: wheels-windows-${{ matrix.platform.target }}
113+
path: dist
114+
115+
macos:
116+
runs-on: ${{ matrix.platform.runner }}
117+
strategy:
118+
matrix:
119+
platform:
120+
- runner: macos-12
121+
target: x86_64
122+
- runner: macos-14
123+
target: aarch64
124+
steps:
125+
- uses: actions/checkout@v4
126+
- uses: actions/setup-python@v5
127+
with:
128+
python-version: 3.x
129+
- name: Build wheels
130+
uses: PyO3/maturin-action@v1
131+
with:
132+
target: ${{ matrix.platform.target }}
133+
args: --release --out dist --manifest-path bindings/python/Cargo.toml
134+
sccache: 'true'
135+
- name: Upload wheels
136+
uses: actions/upload-artifact@v4
137+
with:
138+
name: wheels-macos-${{ matrix.platform.target }}
139+
path: dist
140+
141+
sdist:
142+
runs-on: ubuntu-latest
143+
steps:
144+
- uses: actions/checkout@v4
145+
- name: Build sdist
146+
uses: PyO3/maturin-action@v1
147+
with:
148+
command: sdist
149+
args: --out dist --manifest-path bindings/python/Cargo.toml
150+
- name: Upload sdist
151+
uses: actions/upload-artifact@v4
152+
with:
153+
name: wheels-sdist
154+
path: dist
155+
156+
release:
157+
name: Release
158+
runs-on: ubuntu-latest
159+
if: ${{ startsWith(github.ref, 'refs/tags/') || github.event_name == 'workflow_dispatch' }}
160+
needs: [linux, musllinux, windows, macos, sdist]
161+
permissions:
162+
# Use to sign the release artifacts
163+
id-token: write
164+
# Used to upload release artifacts
165+
contents: write
166+
# Used to generate artifact attestation
167+
attestations: write
168+
steps:
169+
- uses: actions/download-artifact@v4
170+
- name: Generate artifact attestation
171+
uses: actions/attest-build-provenance@v1
172+
with:
173+
subject-path: 'wheels-*/*'
174+
- name: Publish to PyPI
175+
if: "startsWith(github.ref, 'refs/tags/')"
176+
uses: PyO3/maturin-action@v1
177+
env:
178+
MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_TOKEN_DIST}}
179+
with:
180+
command: upload
181+
args: --non-interactive --skip-existing wheels-*/*

‎.github/workflows/python-release.yml

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -164,8 +164,9 @@ jobs:
164164
with:
165165
path: ./bindings/python/dist
166166
merge-multiple: true
167-
- name: Upload to PyPi
168-
working-directory: ./bindings/python
169-
run: |
170-
pip install twine
171-
twine upload dist/* -u __token__ -p "$PYPI_TOKEN"
167+
# Temporary deactivation while testing abi3 CI
168+
# - name: Upload to PyPi
169+
# working-directory: ./bindings/python
170+
# run: |
171+
# pip install twine
172+
# twine upload dist/* -u __token__ -p "$PYPI_TOKEN"

‎.github/workflows/python.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ jobs:
1616
runs-on: windows-latest
1717
strategy:
1818
matrix:
19-
python: ["3.7", "3.8", "3.9", "3.10"]
19+
python: ["3.9", "3.10", "3.11", "3.12", "3.13"]
2020
steps:
2121
- name: Checkout repository
2222
uses: actions/checkout@v4
@@ -72,7 +72,7 @@ jobs:
7272
- name: Install Python
7373
uses: actions/setup-python@v5
7474
with:
75-
python-version: 3.11
75+
python-version: 3.13
7676
architecture: "x64"
7777

7878

‎bindings/python/Cargo.toml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ serde = { version = "1.0", features = ["rc", "derive"] }
1414
serde_json = "1.0"
1515
libc = "0.2"
1616
env_logger = "0.11"
17-
pyo3 = { version = "0.22", features = ["py-clone"] }
17+
pyo3 = { version = "0.22", features = ["abi3", "abi3-py39"] }
1818
numpy = "0.22"
1919
ndarray = "0.15"
2020
itertools = "0.12"
@@ -24,7 +24,7 @@ path = "../../tokenizers"
2424

2525
[dev-dependencies]
2626
tempfile = "3.10"
27-
pyo3 = { version = "0.22", features = ["auto-initialize", "py-clone"] }
27+
pyo3 = { version = "0.22", features = ["auto-initialize"] }
2828

2929
[features]
3030
defaut = ["pyo3/extension-module"]

‎bindings/python/src/decoders.rs

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -488,7 +488,6 @@ impl PySequenceDecoder {
488488
}
489489
}
490490

491-
#[derive(Clone)]
492491
pub(crate) struct CustomDecoder {
493492
inner: PyObject,
494493
}

‎bindings/python/src/encoding.rs

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -399,11 +399,11 @@ impl PyEncoding {
399399

400400
if let Some(kwargs) = kwargs {
401401
for (key, value) in kwargs {
402-
let key: &str = key.extract()?;
403-
match key {
402+
let key: String = key.extract()?;
403+
match key.as_ref() {
404404
"direction" => {
405-
let value: &str = value.extract()?;
406-
direction = match value {
405+
let value: String = value.extract()?;
406+
direction = match value.as_ref() {
407407
"left" => Ok(PaddingDirection::Left),
408408
"right" => Ok(PaddingDirection::Right),
409409
other => Err(PyError(format!(

‎bindings/python/src/models.rs

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -276,8 +276,8 @@ impl PyBPE {
276276
) -> PyResult<(Self, PyModel)> {
277277
if let Some(kwargs) = kwargs {
278278
for (key, value) in kwargs {
279-
let key: &str = key.extract()?;
280-
match key {
279+
let key: String = key.extract()?;
280+
match key.as_ref() {
281281
"cache_capacity" => builder = builder.cache_capacity(value.extract()?),
282282
"dropout" => {
283283
if let Some(dropout) = value.extract()? {
@@ -581,8 +581,8 @@ impl PyWordPiece {
581581
) -> PyResult<(Self, PyModel)> {
582582
if let Some(kwargs) = kwargs {
583583
for (key, val) in kwargs {
584-
let key: &str = key.extract()?;
585-
match key {
584+
let key: String = key.extract()?;
585+
match key.as_ref() {
586586
"unk_token" => {
587587
builder = builder.unk_token(val.extract()?);
588588
}

‎bindings/python/src/normalizers.rs

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -184,9 +184,8 @@ macro_rules! getter {
184184
let super_ = $self.as_ref();
185185
if let PyNormalizerTypeWrapper::Single(ref norm) = super_.normalizer {
186186
let wrapper = norm.read().unwrap();
187-
if let PyNormalizerWrapper::Wrapped(NormalizerWrapper::$variant(o)) = (*wrapper).clone()
188-
{
189-
o.$name
187+
if let PyNormalizerWrapper::Wrapped(NormalizerWrapper::$variant(o)) = (&*wrapper) {
188+
o.$name.clone()
190189
} else {
191190
unreachable!()
192191
}
@@ -538,7 +537,7 @@ impl PyReplace {
538537
}
539538
}
540539

541-
#[derive(Debug, Clone)]
540+
#[derive(Debug)]
542541
pub(crate) struct CustomNormalizer {
543542
inner: PyObject,
544543
}
@@ -581,7 +580,7 @@ impl<'de> Deserialize<'de> for CustomNormalizer {
581580
}
582581
}
583582

584-
#[derive(Debug, Clone, Deserialize)]
583+
#[derive(Debug, Deserialize)]
585584
#[serde(untagged)]
586585
pub(crate) enum PyNormalizerWrapper {
587586
Custom(CustomNormalizer),

‎bindings/python/src/pre_tokenizers.rs

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -618,7 +618,6 @@ impl PyUnicodeScripts {
618618
}
619619
}
620620

621-
#[derive(Clone)]
622621
pub(crate) struct CustomPreTokenizer {
623622
inner: PyObject,
624623
}
@@ -662,7 +661,7 @@ impl<'de> Deserialize<'de> for CustomPreTokenizer {
662661
}
663662
}
664663

665-
#[derive(Clone, Deserialize)]
664+
#[derive(Deserialize)]
666665
#[serde(untagged)]
667666
pub(crate) enum PyPreTokenizerWrapper {
668667
Custom(CustomPreTokenizer),

‎bindings/python/src/processors.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -313,7 +313,7 @@ impl From<PyTemplate> for Template {
313313

314314
impl FromPyObject<'_> for PyTemplate {
315315
fn extract_bound(ob: &Bound<'_, PyAny>) -> PyResult<Self> {
316-
if let Ok(s) = ob.extract::<&str>() {
316+
if let Ok(s) = ob.extract::<String>() {
317317
Ok(Self(
318318
s.try_into().map_err(exceptions::PyValueError::new_err)?,
319319
))

0 commit comments

Comments
 (0)