Skip to content

Commit 5c986fe

Browse files
booxterbooxter
committed
ci: Constrain all dependencies
Signed-off-by: Ihar Hrachyshka <[email protected]>
1 parent 547a820 commit 5c986fe

17 files changed

+353
-87
lines changed

‎.github/actions/run-smoke/action.yml

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ runs:
2929
run: |
3030
python${{ inputs.python-version }} -m venv --upgrade-deps venv
3131
. venv/bin/activate
32-
pip install tox
32+
pip install tox -c constraints-dev.txt
3333
3434
# flash-attn has a bug in the setup.py that causes pip to attempt
3535
# installing it before torch is installed. This is a bug because their
@@ -41,7 +41,7 @@ runs:
4141
run: |
4242
source venv/bin/activate
4343
# The list is taken from the pull request linked above
44-
pip install torch packaging setuptools wheel psutil ninja
44+
pip install torch packaging setuptools wheel psutil ninja -c constraints-dev.txt
4545
4646
- name: "Install tox-current-env to reuse the venv with pre-installed build dependencies"
4747
shell: bash
@@ -54,8 +54,9 @@ runs:
5454
run: |
5555
source venv/bin/activate
5656
tox -e py3-smoke --print-deps-to-file=./deps.txt
57-
pip install -r ./deps.txt --no-build-isolation
58-
pip install .
57+
pip_install="pip install -c constraints-dev.txt"
58+
$pip_install -r ./deps.txt --no-build-isolation
59+
$pip_install .
5960
6061
- name: "Show disk utilization BEFORE tests"
6162
shell: bash

‎.github/workflows/constraints-update.yml

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
name: Update constraints-dev.txt
2+
3+
on:
4+
schedule:
5+
- cron: '0 3 * * 1' # Every Monday at 03:00 UTC
6+
workflow_dispatch:
7+
8+
jobs:
9+
update-constraints:
10+
runs-on: ubuntu-latest
11+
permissions:
12+
contents: write
13+
pull-requests: write
14+
15+
steps:
16+
- name: Checkout repository
17+
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
18+
19+
- name: Checkout "update-constraints" in-house CI action
20+
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
21+
with:
22+
repository: instructlab/ci-actions
23+
path: ci-actions
24+
ref: v0.2.0
25+
sparse-checkout: |
26+
actions/update-constraints
27+
28+
- name: Update constraints
29+
id: update-constraints
30+
uses: ./ci-actions/actions/update-constraints
31+
with:
32+
gh-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
33+

‎.github/workflows/e2e-nvidia-l40s-x4.yml

Lines changed: 18 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -177,27 +177,28 @@ jobs:
177177
- name: Install ilab
178178
working-directory: ./instructlab
179179
run: |
180-
export CUDA_HOME="/usr/local/cuda"
181-
export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64"
182-
export PATH="$PATH:$CUDA_HOME/bin"
183-
python3.11 -m venv --upgrade-deps venv
184-
. venv/bin/activate
185-
nvidia-smi
186-
python3.11 -m pip cache remove llama_cpp_python
187-
188-
CMAKE_ARGS="-DLLAMA_CUDA=on" python3.11 -m pip install . -c constraints-dev.txt
189-
190-
# https://github.com/instructlab/instructlab/issues/1821
191-
# install with Torch and build dependencies installed
192-
python3.11 -m pip install packaging wheel setuptools-scm
193-
python3.11 -m pip install .[cuda] -r requirements-vllm-cuda.txt
180+
PYTHON=python3.11 ./scripts/install-ilab-with-cuda.sh
194181
195182
- name: Update instructlab-training library
196183
working-directory: ./training
197184
run: |
198185
. ../instructlab/venv/bin/activate
199-
pip install .
200-
pip install .[cuda]
186+
# Patch out our own pin from the ilab repo constraints file
187+
ilab_constraints=../instructlab/constraints-dev.txt
188+
sed -i '/instructlab-training==/d' $ilab_constraints
189+
# Since we reuse the virtual environment prepared using ilab
190+
# constraints, we should stick to the same constraints when
191+
# installing latest training.
192+
#
193+
# FIX: this is not ideal; a proper fix would require decoupling the
194+
# two repos in CI: either by removing the job completely and relying
195+
# on "sdk" (no ilab) test runs; or by preparing a separate
196+
# constraints file that would consider both the requirements files
197+
# for the training library AND for the ilab - so that they are
198+
# consistent.
199+
pip_install="pip install -c $ilab_constraints"
200+
$pip_install .
201+
$pip_install .[cuda]
201202
202203
- name: Check disk before tests
203204
if: always()
@@ -380,7 +381,7 @@ jobs:
380381
working-directory: ./training
381382
run: |
382383
python -m pip install --upgrade pip
383-
pip install -r requirements-dev.txt
384+
pip install -r requirements-dev.txt -c constraints-dev.txt
384385
385386
- name: Try to upload Phase 1 to s3
386387
id: phase-1-upload-s3

‎.github/workflows/lint.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,7 @@ jobs:
8080
- name: Install tox
8181
run: |
8282
python -m pip install --upgrade pip
83-
python -m pip install tox tox-gh
83+
python -m pip install tox tox-gh -c constraints-dev.txt
8484
8585
- name: "${{ matrix.lint.name }}"
8686
run: |

‎.github/workflows/unit.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,7 @@ jobs:
7272
run: |
7373
python -m venv --upgrade-deps venv
7474
. venv/bin/activate
75-
pip install tox
75+
pip install tox -c constraints-dev.txt
7676
7777
- name: "Show disk utilization BEFORE tests"
7878
if: always()

‎constraints-dev.txt

Lines changed: 193 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,193 @@
1+
accelerate==1.0.1 # via -r requirements-cuda.txt, -r requirements-rocm.txt, peft, trl
2+
aiofiles==24.1.0 # via -r requirements.txt
3+
aiohappyeyeballs==2.6.1 # via aiohttp
4+
aiohttp==3.12.2 # via fsspec
5+
aiosignal==1.3.2 # via aiohttp
6+
annotated-types==0.7.0 # via pydantic
7+
anyio==4.9.0 # via httpx, jupyter-server
8+
argon2-cffi==23.1.0 # via jupyter-server
9+
argon2-cffi-bindings==21.2.0 # via argon2-cffi
10+
arrow==1.3.0 # via isoduration
11+
astroid==3.3.10 # via pylint
12+
asttokens==3.0.0 # via stack-data
13+
async-lru==2.0.5 # via jupyterlab
14+
attrs==25.3.0 # via aiohttp, jsonschema, referencing
15+
babel==2.17.0 # via jupyterlab-server
16+
beautifulsoup4==4.13.4 # via nbconvert
17+
bitsandbytes==0.46.0 # via -r requirements-cuda.txt
18+
bleach==6.2.0 # via nbconvert
19+
cachetools==6.0.0 # via tox
20+
certifi==2025.4.26 # via httpcore, httpx, requests
21+
cffi==1.17.1 # via argon2-cffi-bindings
22+
cfgv==3.4.0 # via pre-commit
23+
chardet==5.2.0 # via tox
24+
charset-normalizer==3.4.2 # via requests
25+
colorama==0.4.6 # via tox
26+
comm==0.2.2 # via ipykernel, ipywidgets
27+
contourpy==1.3.2 # via matplotlib
28+
cycler==0.12.1 # via matplotlib
29+
datasets==3.6.0 # via -r requirements.txt, trl
30+
debugpy==1.8.14 # via ipykernel
31+
decorator==5.2.1 # via ipython
32+
deepspeed==0.16.9 # via -r requirements-deepspeed.txt
33+
defusedxml==0.7.1 # via nbconvert
34+
dill==0.3.8 # via datasets, multiprocess, pylint
35+
distlib==0.3.9 # via virtualenv
36+
einops==0.8.1 # via deepspeed, flash-attn
37+
executing==2.2.0 # via stack-data
38+
fastjsonschema==2.21.1 # via nbformat
39+
filelock==3.18.0 # via datasets, huggingface-hub, torch, tox, transformers, virtualenv
40+
flash-attn==2.7.4.post1 # via -r requirements-cuda.txt, -r requirements-rocm.txt
41+
fonttools==4.58.1 # via matplotlib
42+
fqdn==1.5.1 # via jsonschema
43+
frozenlist==1.6.0 # via aiohttp, aiosignal
44+
fsspec==2025.3.0 # via datasets, huggingface-hub, torch
45+
h11==0.16.0 # via httpcore
46+
hf-xet==1.1.2 # via huggingface-hub
47+
hjson==3.1.0 # via deepspeed
48+
httpcore==1.0.9 # via httpx
49+
httpx==0.28.1 # via jupyterlab
50+
huggingface-hub==0.32.2 # via -r requirements-dev.txt, accelerate, datasets, peft, tokenizers, transformers
51+
identify==2.6.12 # via pre-commit
52+
idna==3.10 # via anyio, httpx, jsonschema, requests, yarl
53+
instructlab-dolomite==0.2.0 # via -r requirements.txt
54+
ipykernel==6.29.5 # via -r requirements-dev.txt, jupyter, jupyter-console, jupyterlab
55+
ipython==9.2.0 # via -r requirements-dev.txt, ipykernel, ipywidgets, jupyter-console
56+
ipython-pygments-lexers==1.1.1 # via ipython
57+
ipywidgets==8.1.7 # via jupyter
58+
isoduration==20.11.0 # via jsonschema
59+
isort==6.0.1 # via -r requirements-dev.txt, pylint
60+
jedi==0.19.2 # via ipython
61+
jinja2==3.1.6 # via jupyter-server, jupyterlab, jupyterlab-server, nbconvert, torch
62+
json5==0.12.0 # via jupyterlab-server
63+
jsonpointer==3.0.0 # via jsonschema
64+
jsonschema==4.24.0 # via jupyter-events, jupyterlab-server, nbformat
65+
jsonschema-specifications==2025.4.1 # via jsonschema
66+
jupyter==1.1.1 # via -r requirements-dev.txt
67+
jupyter-client==8.6.3 # via ipykernel, jupyter-console, jupyter-server, nbclient
68+
jupyter-console==6.6.3 # via jupyter
69+
jupyter-core==5.8.1 # via ipykernel, jupyter-client, jupyter-console, jupyter-server, jupyterlab, nbclient, nbconvert, nbformat
70+
jupyter-events==0.12.0 # via jupyter-server
71+
jupyter-lsp==2.2.5 # via jupyterlab
72+
jupyter-server==2.16.0 # via jupyter-lsp, jupyterlab, jupyterlab-server, notebook, notebook-shim
73+
jupyter-server-terminals==0.5.3 # via jupyter-server
74+
jupyterlab==4.4.3 # via jupyter, notebook
75+
jupyterlab-pygments==0.3.0 # via nbconvert
76+
jupyterlab-server==2.27.3 # via jupyterlab, notebook
77+
jupyterlab-widgets==3.0.15 # via ipywidgets
78+
kiwisolver==1.4.8 # via matplotlib
79+
liger-kernel==0.5.10 # via -r requirements-cuda.txt
80+
llvmlite==0.44.0 # via numba
81+
markdown-it-py==3.0.0 # via rich
82+
markupsafe==3.0.2 # via jinja2, nbconvert
83+
matplotlib==3.10.3 # via -r requirements-dev.txt
84+
matplotlib-inline==0.1.7 # via ipykernel, ipython
85+
mccabe==0.7.0 # via pylint
86+
mdurl==0.1.2 # via markdown-it-py
87+
mistune==3.1.3 # via nbconvert
88+
mpmath==1.3.0 # via sympy
89+
msgpack==1.1.0 # via deepspeed
90+
multidict==6.4.4 # via aiohttp, yarl
91+
multiprocess==0.70.16 # via datasets
92+
nbclient==0.10.2 # via nbconvert
93+
nbconvert==7.16.6 # via jupyter, jupyter-server
94+
nbformat==5.10.4 # via jupyter-server, nbclient, nbconvert
95+
nest-asyncio==1.6.0 # via ipykernel
96+
networkx==3.4.2 # via torch
97+
ninja==1.11.1.4 # via deepspeed
98+
nodeenv==1.9.1 # via pre-commit
99+
notebook==7.4.3 # via jupyter
100+
notebook-shim==0.2.4 # via jupyterlab, notebook
101+
numba==0.61.2 # via -r requirements.txt
102+
numpy==1.26.4 # via -r requirements-dev.txt, -r requirements.txt, accelerate, bitsandbytes, contourpy, datasets, deepspeed, matplotlib, numba, pandas, peft, transformers
103+
nvidia-cublas-cu12==12.4.5.8 # via nvidia-cudnn-cu12, nvidia-cusolver-cu12, torch
104+
nvidia-cuda-cupti-cu12==12.4.127 # via torch
105+
nvidia-cuda-nvrtc-cu12==12.4.127 # via torch
106+
nvidia-cuda-runtime-cu12==12.4.127 # via torch
107+
nvidia-cudnn-cu12==9.1.0.70 # via torch
108+
nvidia-cufft-cu12==11.2.1.3 # via torch
109+
nvidia-curand-cu12==10.3.5.147 # via torch
110+
nvidia-cusolver-cu12==11.6.1.9 # via torch
111+
nvidia-cusparse-cu12==12.3.1.170 # via nvidia-cusolver-cu12, torch
112+
nvidia-cusparselt-cu12==0.6.2 # via torch
113+
nvidia-ml-py==12.575.51 # via deepspeed
114+
nvidia-nccl-cu12==2.21.5 # via torch
115+
nvidia-nvjitlink-cu12==12.4.127 # via nvidia-cusolver-cu12, nvidia-cusparse-cu12, torch
116+
nvidia-nvtx-cu12==12.4.127 # via torch
117+
overrides==7.7.0 # via jupyter-server
118+
packaging==25.0 # via -r requirements.txt, accelerate, datasets, deepspeed, huggingface-hub, ipykernel, jupyter-events, jupyter-server, jupyterlab, jupyterlab-server, matplotlib, nbconvert, peft, pyproject-api, tox, transformers
119+
pandas==2.2.3 # via datasets
120+
pandocfilters==1.5.1 # via nbconvert
121+
parso==0.8.4 # via jedi
122+
peft==0.15.2 # via -r requirements.txt
123+
pexpect==4.9.0 # via ipython
124+
pillow==11.2.1 # via matplotlib
125+
platformdirs==4.3.8 # via jupyter-core, pylint, tox, virtualenv
126+
pluggy==1.6.0 # via tox
127+
pre-commit==4.2.0 # via -r requirements-dev.txt
128+
prometheus-client==0.22.0 # via jupyter-server
129+
prompt-toolkit==3.0.51 # via ipython, jupyter-console
130+
propcache==0.3.1 # via aiohttp, yarl
131+
psutil==7.0.0 # via accelerate, deepspeed, ipykernel, peft
132+
ptyprocess==0.7.0 # via pexpect, terminado
133+
pure-eval==0.2.3 # via stack-data
134+
py-cpuinfo==9.0.0 # via -r requirements.txt, deepspeed
135+
pyarrow==20.0.0 # via datasets
136+
pycparser==2.22 # via cffi
137+
pydantic==2.11.5 # via -r requirements.txt, deepspeed, pylint-pydantic
138+
pydantic-core==2.33.2 # via pydantic
139+
pygments==2.19.1 # via ipython, ipython-pygments-lexers, jupyter-console, nbconvert, rich
140+
pylint==3.3.7 # via -r requirements-dev.txt, pylint-plugin-utils, pylint-pydantic
141+
pylint-plugin-utils==0.8.2 # via pylint-pydantic
142+
pylint-pydantic==0.3.5 # via -r requirements-dev.txt
143+
pyparsing==3.2.3 # via matplotlib
144+
pyproject-api==1.9.1 # via tox
145+
python-dateutil==2.9.0.post0 # via arrow, jupyter-client, matplotlib, pandas
146+
python-json-logger==3.3.0 # via jupyter-events
147+
pytz==2025.2 # via pandas
148+
pyyaml==6.0.2 # via -r requirements.txt, accelerate, datasets, huggingface-hub, jupyter-events, peft, pre-commit, transformers
149+
pyzmq==26.4.0 # via ipykernel, jupyter-client, jupyter-console, jupyter-server
150+
referencing==0.36.2 # via jsonschema, jsonschema-specifications, jupyter-events
151+
regex==2024.11.6 # via transformers
152+
requests==2.32.3 # via datasets, huggingface-hub, jupyterlab-server, transformers
153+
rfc3339-validator==0.1.4 # via jsonschema, jupyter-events
154+
rfc3986-validator==0.1.1 # via jsonschema, jupyter-events
155+
rich==14.0.0 # via -r requirements.txt
156+
rpds-py==0.25.1 # via jsonschema, referencing
157+
ruff==0.11.11 # via -r requirements-dev.txt
158+
safetensors==0.5.3 # via accelerate, instructlab-dolomite, peft, transformers
159+
send2trash==1.8.3 # via jupyter-server
160+
six==1.17.0 # via python-dateutil, rfc3339-validator
161+
sniffio==1.3.1 # via anyio
162+
soupsieve==2.7 # via beautifulsoup4
163+
stack-data==0.6.3 # via ipython
164+
sympy==1.13.1 # via torch
165+
terminado==0.18.1 # via jupyter-server, jupyter-server-terminals
166+
tinycss2==1.4.0 # via bleach
167+
tokenizers==0.21.1 # via transformers
168+
tomlkit==0.13.2 # via pylint
169+
torch==2.6.0 # via -c constraints-dev.txt.in, -r requirements.txt, accelerate, bitsandbytes, deepspeed, flash-attn, instructlab-dolomite, liger-kernel, peft
170+
tornado==6.5.1 # via ipykernel, jupyter-client, jupyter-server, jupyterlab, notebook, terminado
171+
tox==4.26.0 # via -r requirements-dev.txt, tox-current-env
172+
tox-current-env==0.0.16 # via -r requirements-dev.txt
173+
tqdm==4.67.1 # via datasets, deepspeed, huggingface-hub, peft, transformers
174+
traitlets==5.14.3 # via comm, ipykernel, ipython, ipywidgets, jupyter-client, jupyter-console, jupyter-core, jupyter-events, jupyter-server, jupyterlab, matplotlib-inline, nbclient, nbconvert, nbformat
175+
transformers==4.52.3 # via -r requirements.txt, instructlab-dolomite, peft, trl
176+
triton==3.2.0 # via liger-kernel, torch
177+
trl==0.18.0 # via -r requirements.txt
178+
types-python-dateutil==2.9.0.20250516 # via arrow
179+
typing-extensions==4.13.2 # via anyio, beautifulsoup4, huggingface-hub, ipython, pydantic, pydantic-core, referencing, torch, typing-inspection
180+
typing-inspection==0.4.1 # via pydantic
181+
tzdata==2025.2 # via pandas
182+
uri-template==1.3.0 # via jsonschema
183+
urllib3==2.4.0 # via requests
184+
virtualenv==20.31.2 # via pre-commit, tox
185+
wcwidth==0.2.13 # via prompt-toolkit
186+
webcolors==24.11.1 # via jsonschema
187+
webencodings==0.5.1 # via bleach, tinycss2
188+
websocket-client==1.8.0 # via jupyter-server
189+
wheel==0.45.1 # via -r requirements.txt
190+
widgetsnbextension==4.0.14 # via ipywidgets
191+
xxhash==3.5.0 # via datasets
192+
yarl==1.20.0 # via aiohttp
193+
setuptools==80.9.0 # via jupyterlab

‎constraints-dev.txt.in

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
# SPDX-License-Identifier: Apache-2.0
2+
3+
# These are synchronized with instructlab repo; we have to keep them in sync at
4+
least until we no longer tie training repo CI with ilab repo through e2e jobs.
5+
torch<2.7.0
6+
vllm<0.9.0

0 commit comments

Comments
 (0)