Skip to content

Commit 861835c

Browse files
booxterbooxter
committed
ci: Constrain all dependencies
Signed-off-by: Ihar Hrachyshka <[email protected]>
1 parent 1532531 commit 861835c

17 files changed

+372
-87
lines changed

‎.github/actions/run-smoke/action.yml

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ runs:
3434
run: |
3535
python${{ inputs.python-version }} -m venv --upgrade-deps venv
3636
. venv/bin/activate
37-
pip install tox
37+
pip install tox -c constraints-dev.txt
3838
3939
# flash-attn has a bug in the setup.py that causes pip to attempt
4040
# installing it before torch is installed. This is a bug because their
@@ -46,7 +46,7 @@ runs:
4646
run: |
4747
source venv/bin/activate
4848
# The list is taken from the pull request linked above
49-
pip install torch packaging setuptools wheel psutil ninja
49+
pip install torch packaging setuptools wheel psutil ninja -c constraints-dev.txt
5050
5151
- name: "Install tox-current-env to reuse the venv with pre-installed build dependencies"
5252
shell: bash
@@ -59,8 +59,9 @@ runs:
5959
run: |
6060
source venv/bin/activate
6161
tox -e py3-smoke --print-deps-to-file=./deps.txt
62-
pip install -r ./deps.txt --no-build-isolation
63-
pip install .
62+
pip_install="pip install -c constraints-dev.txt"
63+
$pip_install -r ./deps.txt --no-build-isolation
64+
$pip_install .
6465
6566
- name: "Show disk utilization BEFORE tests"
6667
shell: bash

‎.github/workflows/constraints-update.yml

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
name: Update constraints-dev.txt
2+
3+
on:
4+
schedule:
5+
- cron: '0 3 * * 1' # Every Monday at 03:00 UTC
6+
workflow_dispatch:
7+
8+
jobs:
9+
update-constraints:
10+
runs-on: ubuntu-latest
11+
permissions:
12+
contents: write
13+
pull-requests: write
14+
15+
steps:
16+
- name: Checkout repository
17+
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
18+
19+
- name: Set up Python
20+
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
21+
with:
22+
python-version: '3.11'
23+
24+
- name: Install tox
25+
run: python -m pip install --upgrade pip tox -c constraints-dev.txt
26+
27+
- name: Run tox constraints
28+
run: tox -e constraints
29+
30+
- name: git diff
31+
run: |
32+
git diff constraints-dev.txt
33+
34+
- name: set release date
35+
run: |
36+
echo "date=$(date --rfc-3339=date)" >> "${GITHUB_ENV}"
37+
38+
- name: Create Pull Request
39+
uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e # v7.0.8
40+
with:
41+
signoff: true
42+
commit-message: "chore: update constraints-dev.txt (${{ env.date }})"
43+
title: "chore: update constraints-dev.txt (${{ env.date }})"
44+
body: |
45+
This PR updates the constraints-dev.txt file using `tox -e constraints`.
46+
branch: update-constraints
47+
token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}

‎.github/workflows/e2e-nvidia-l40s-x4.yml

Lines changed: 5 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -177,27 +177,15 @@ jobs:
177177
- name: Install ilab
178178
working-directory: ./instructlab
179179
run: |
180-
export CUDA_HOME="/usr/local/cuda"
181-
export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64"
182-
export PATH="$PATH:$CUDA_HOME/bin"
183-
python3.11 -m venv --upgrade-deps venv
184-
. venv/bin/activate
185-
nvidia-smi
186-
python3.11 -m pip cache remove llama_cpp_python
187-
188-
CMAKE_ARGS="-DLLAMA_CUDA=on" python3.11 -m pip install . -c constraints-dev.txt
189-
190-
# https://github.com/instructlab/instructlab/issues/1821
191-
# install with Torch and build dependencies installed
192-
python3.11 -m pip install packaging wheel setuptools-scm
193-
python3.11 -m pip install .[cuda] -r requirements-vllm-cuda.txt
180+
./scripts/install-ilab-with-cuda.sh
194181
195182
- name: Update instructlab-training library
196183
working-directory: ./training
197184
run: |
198185
. ../instructlab/venv/bin/activate
199-
pip install .
200-
pip install .[cuda]
186+
pip_install="pip install -c constraints-dev.txt"
187+
$pip_install .
188+
$pip_install .[cuda]
201189
202190
- name: Check disk before tests
203191
if: always()
@@ -380,7 +368,7 @@ jobs:
380368
working-directory: ./training
381369
run: |
382370
python -m pip install --upgrade pip
383-
pip install -r requirements-dev.txt
371+
pip install -r requirements-dev.txt -c constraints-dev.txt
384372
385373
- name: Try to upload Phase 1 to s3
386374
id: phase-1-upload-s3

‎.github/workflows/lint.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,7 @@ jobs:
8080
- name: Install tox
8181
run: |
8282
python -m pip install --upgrade pip
83-
python -m pip install tox tox-gh
83+
python -m pip install tox tox-gh -c constraints-dev.txt
8484
8585
- name: "${{ matrix.lint.name }}"
8686
run: |

‎.github/workflows/unit.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,7 @@ jobs:
7272
run: |
7373
python -m venv --upgrade-deps venv
7474
. venv/bin/activate
75-
pip install tox
75+
pip install tox -c constraints-dev.txt
7676
7777
- name: "Show disk utilization BEFORE tests"
7878
if: always()

‎constraints-dev.txt

Lines changed: 193 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,193 @@
1+
accelerate==1.0.1 # via -r requirements-cuda.txt, -r requirements-rocm.txt, peft, trl
2+
aiofiles==24.1.0 # via -r requirements.txt
3+
aiohappyeyeballs==2.6.1 # via aiohttp
4+
aiohttp==3.11.18 # via fsspec
5+
aiosignal==1.3.2 # via aiohttp
6+
annotated-types==0.7.0 # via pydantic
7+
anyio==4.9.0 # via httpx, jupyter-server
8+
argon2-cffi==23.1.0 # via jupyter-server
9+
argon2-cffi-bindings==21.2.0 # via argon2-cffi
10+
arrow==1.3.0 # via isoduration
11+
astroid==3.3.10 # via pylint
12+
asttokens==3.0.0 # via stack-data
13+
async-lru==2.0.5 # via jupyterlab
14+
attrs==25.3.0 # via aiohttp, jsonschema, referencing
15+
babel==2.17.0 # via jupyterlab-server
16+
beautifulsoup4==4.13.4 # via nbconvert
17+
bitsandbytes==0.45.5 # via -r requirements-cuda.txt
18+
bleach==6.2.0 # via nbconvert
19+
cachetools==5.5.2 # via tox
20+
certifi==2025.4.26 # via httpcore, httpx, requests
21+
cffi==1.17.1 # via argon2-cffi-bindings
22+
cfgv==3.4.0 # via pre-commit
23+
chardet==5.2.0 # via tox
24+
charset-normalizer==3.4.2 # via requests
25+
colorama==0.4.6 # via tox
26+
comm==0.2.2 # via ipykernel, ipywidgets
27+
contourpy==1.3.2 # via matplotlib
28+
cycler==0.12.1 # via matplotlib
29+
datasets==3.6.0 # via -r requirements.txt, trl
30+
debugpy==1.8.14 # via ipykernel
31+
decorator==5.2.1 # via ipython
32+
deepspeed==0.16.8 # via -r requirements-deepspeed.txt
33+
defusedxml==0.7.1 # via nbconvert
34+
dill==0.3.8 # via datasets, multiprocess, pylint
35+
distlib==0.3.9 # via virtualenv
36+
einops==0.8.1 # via deepspeed, flash-attn
37+
executing==2.2.0 # via stack-data
38+
fastjsonschema==2.21.1 # via nbformat
39+
filelock==3.18.0 # via datasets, huggingface-hub, torch, tox, transformers, virtualenv
40+
flash-attn==2.7.4.post1 # via -r requirements-cuda.txt, -r requirements-rocm.txt
41+
fonttools==4.58.0 # via matplotlib
42+
fqdn==1.5.1 # via jsonschema
43+
frozenlist==1.6.0 # via aiohttp, aiosignal
44+
fsspec==2025.3.0 # via datasets, huggingface-hub, torch
45+
h11==0.16.0 # via httpcore
46+
hjson==3.1.0 # via deepspeed
47+
httpcore==1.0.9 # via httpx
48+
httpx==0.28.1 # via jupyterlab
49+
huggingface-hub==0.31.4 # via -r requirements-dev.txt, accelerate, datasets, peft, tokenizers, transformers
50+
identify==2.6.10 # via pre-commit
51+
idna==3.10 # via anyio, httpx, jsonschema, requests, yarl
52+
instructlab-dolomite==0.2.0 # via -r requirements.txt
53+
ipykernel==6.29.5 # via -r requirements-dev.txt, jupyter, jupyter-console, jupyterlab
54+
ipython==9.2.0 # via -r requirements-dev.txt, ipykernel, ipywidgets, jupyter-console
55+
ipython-pygments-lexers==1.1.1 # via ipython
56+
ipywidgets==8.1.7 # via jupyter
57+
isoduration==20.11.0 # via jsonschema
58+
isort==6.0.1 # via -r requirements-dev.txt, pylint
59+
jedi==0.19.2 # via ipython
60+
jinja2==3.1.6 # via jupyter-server, jupyterlab, jupyterlab-server, nbconvert, torch
61+
json5==0.12.0 # via jupyterlab-server
62+
jsonpointer==3.0.0 # via jsonschema
63+
jsonschema==4.23.0 # via jupyter-events, jupyterlab-server, nbformat
64+
jsonschema-specifications==2025.4.1 # via jsonschema
65+
jupyter==1.1.1 # via -r requirements-dev.txt
66+
jupyter-client==8.6.3 # via ipykernel, jupyter-console, jupyter-server, nbclient
67+
jupyter-console==6.6.3 # via jupyter
68+
jupyter-core==5.7.2 # via ipykernel, jupyter-client, jupyter-console, jupyter-server, jupyterlab, nbclient, nbconvert, nbformat
69+
jupyter-events==0.12.0 # via jupyter-server
70+
jupyter-lsp==2.2.5 # via jupyterlab
71+
jupyter-server==2.16.0 # via jupyter-lsp, jupyterlab, jupyterlab-server, notebook, notebook-shim
72+
jupyter-server-terminals==0.5.3 # via jupyter-server
73+
jupyterlab==4.4.2 # via jupyter, notebook
74+
jupyterlab-pygments==0.3.0 # via nbconvert
75+
jupyterlab-server==2.27.3 # via jupyterlab, notebook
76+
jupyterlab-widgets==3.0.15 # via ipywidgets
77+
kiwisolver==1.4.8 # via matplotlib
78+
liger-kernel==0.5.9 # via -r requirements-cuda.txt
79+
llvmlite==0.44.0 # via numba
80+
markdown-it-py==3.0.0 # via rich
81+
markupsafe==3.0.2 # via jinja2, nbconvert
82+
matplotlib==3.10.3 # via -r requirements-dev.txt
83+
matplotlib-inline==0.1.7 # via ipykernel, ipython
84+
mccabe==0.7.0 # via pylint
85+
mdurl==0.1.2 # via markdown-it-py
86+
mistune==3.1.3 # via nbconvert
87+
mpmath==1.3.0 # via sympy
88+
msgpack==1.1.0 # via deepspeed
89+
multidict==6.4.4 # via aiohttp, yarl
90+
multiprocess==0.70.16 # via datasets
91+
nbclient==0.10.2 # via nbconvert
92+
nbconvert==7.16.6 # via jupyter, jupyter-server
93+
nbformat==5.10.4 # via jupyter-server, nbclient, nbconvert
94+
nest-asyncio==1.6.0 # via ipykernel
95+
networkx==3.4.2 # via torch
96+
ninja==1.11.1.4 # via deepspeed
97+
nodeenv==1.9.1 # via pre-commit
98+
notebook==7.4.2 # via jupyter
99+
notebook-shim==0.2.4 # via jupyterlab, notebook
100+
numba==0.61.2 # via -r requirements.txt
101+
numpy==1.26.4 # via -r requirements-dev.txt, -r requirements.txt, accelerate, bitsandbytes, contourpy, datasets, deepspeed, matplotlib, numba, pandas, peft, transformers
102+
nvidia-cublas-cu12==12.6.4.1 # via nvidia-cudnn-cu12, nvidia-cusolver-cu12, torch
103+
nvidia-cuda-cupti-cu12==12.6.80 # via torch
104+
nvidia-cuda-nvrtc-cu12==12.6.77 # via torch
105+
nvidia-cuda-runtime-cu12==12.6.77 # via torch
106+
nvidia-cudnn-cu12==9.5.1.17 # via torch
107+
nvidia-cufft-cu12==11.3.0.4 # via torch
108+
nvidia-cufile-cu12==1.11.1.6 # via torch
109+
nvidia-curand-cu12==10.3.7.77 # via torch
110+
nvidia-cusolver-cu12==11.7.1.2 # via torch
111+
nvidia-cusparse-cu12==12.5.4.2 # via nvidia-cusolver-cu12, torch
112+
nvidia-cusparselt-cu12==0.6.3 # via torch
113+
nvidia-ml-py==12.575.51 # via deepspeed
114+
nvidia-nccl-cu12==2.26.2 # via torch
115+
nvidia-nvjitlink-cu12==12.6.85 # via nvidia-cufft-cu12, nvidia-cusolver-cu12, nvidia-cusparse-cu12, torch
116+
nvidia-nvtx-cu12==12.6.77 # via torch
117+
overrides==7.7.0 # via jupyter-server
118+
packaging==25.0 # via -r requirements.txt, accelerate, datasets, deepspeed, huggingface-hub, ipykernel, jupyter-events, jupyter-server, jupyterlab, jupyterlab-server, matplotlib, nbconvert, peft, pyproject-api, tox, transformers
119+
pandas==2.2.3 # via datasets
120+
pandocfilters==1.5.1 # via nbconvert
121+
parso==0.8.4 # via jedi
122+
peft==0.15.2 # via -r requirements.txt
123+
pexpect==4.9.0 # via ipython
124+
pillow==11.2.1 # via matplotlib
125+
platformdirs==4.3.8 # via jupyter-core, pylint, tox, virtualenv
126+
pluggy==1.6.0 # via tox
127+
pre-commit==4.2.0 # via -r requirements-dev.txt
128+
prometheus-client==0.22.0 # via jupyter-server
129+
prompt-toolkit==3.0.51 # via ipython, jupyter-console
130+
propcache==0.3.1 # via aiohttp, yarl
131+
psutil==7.0.0 # via accelerate, deepspeed, ipykernel, peft
132+
ptyprocess==0.7.0 # via pexpect, terminado
133+
pure-eval==0.2.3 # via stack-data
134+
py-cpuinfo==9.0.0 # via -r requirements.txt, deepspeed
135+
pyarrow==20.0.0 # via datasets
136+
pycparser==2.22 # via cffi
137+
pydantic==2.11.4 # via -r requirements.txt, deepspeed, pylint-pydantic
138+
pydantic-core==2.33.2 # via pydantic
139+
pygments==2.19.1 # via ipython, ipython-pygments-lexers, jupyter-console, nbconvert, rich
140+
pylint==3.3.7 # via -r requirements-dev.txt, pylint-plugin-utils, pylint-pydantic
141+
pylint-plugin-utils==0.8.2 # via pylint-pydantic
142+
pylint-pydantic==0.3.5 # via -r requirements-dev.txt
143+
pyparsing==3.2.3 # via matplotlib
144+
pyproject-api==1.9.1 # via tox
145+
python-dateutil==2.9.0.post0 # via arrow, jupyter-client, matplotlib, pandas
146+
python-json-logger==3.3.0 # via jupyter-events
147+
pytz==2025.2 # via pandas
148+
pyyaml==6.0.2 # via -r requirements.txt, accelerate, datasets, huggingface-hub, jupyter-events, peft, pre-commit, transformers
149+
pyzmq==26.4.0 # via ipykernel, jupyter-client, jupyter-console, jupyter-server
150+
referencing==0.36.2 # via jsonschema, jsonschema-specifications, jupyter-events
151+
regex==2024.11.6 # via transformers
152+
requests==2.32.3 # via datasets, huggingface-hub, jupyterlab-server, transformers
153+
rfc3339-validator==0.1.4 # via jsonschema, jupyter-events
154+
rfc3986-validator==0.1.1 # via jsonschema, jupyter-events
155+
rich==14.0.0 # via -r requirements.txt, trl
156+
rpds-py==0.25.1 # via jsonschema, referencing
157+
ruff==0.11.10 # via -r requirements-dev.txt
158+
safetensors==0.5.3 # via accelerate, instructlab-dolomite, peft, transformers
159+
send2trash==1.8.3 # via jupyter-server
160+
six==1.17.0 # via python-dateutil, rfc3339-validator
161+
sniffio==1.3.1 # via anyio
162+
soupsieve==2.7 # via beautifulsoup4
163+
stack-data==0.6.3 # via ipython
164+
sympy==1.14.0 # via torch
165+
terminado==0.18.1 # via jupyter-server, jupyter-server-terminals
166+
tinycss2==1.4.0 # via bleach
167+
tokenizers==0.21.1 # via transformers
168+
tomlkit==0.13.2 # via pylint
169+
torch==2.7.0 # via -r requirements.txt, accelerate, bitsandbytes, deepspeed, flash-attn, instructlab-dolomite, liger-kernel, peft
170+
tornado==6.5 # via ipykernel, jupyter-client, jupyter-server, jupyterlab, notebook, terminado
171+
tox==4.26.0 # via -r requirements-dev.txt, tox-current-env
172+
tox-current-env==0.0.16 # via -r requirements-dev.txt
173+
tqdm==4.67.1 # via datasets, deepspeed, huggingface-hub, peft, transformers
174+
traitlets==5.14.3 # via comm, ipykernel, ipython, ipywidgets, jupyter-client, jupyter-console, jupyter-core, jupyter-events, jupyter-server, jupyterlab, matplotlib-inline, nbclient, nbconvert, nbformat
175+
transformers==4.52.3 # via -r requirements.txt, instructlab-dolomite, peft, trl
176+
triton==3.3.0 # via liger-kernel, torch
177+
trl==0.17.0 # via -r requirements.txt
178+
types-python-dateutil==2.9.0.20250516 # via arrow
179+
typing-extensions==4.13.2 # via anyio, beautifulsoup4, huggingface-hub, ipython, pydantic, pydantic-core, referencing, torch, typing-inspection
180+
typing-inspection==0.4.1 # via pydantic
181+
tzdata==2025.2 # via pandas
182+
uri-template==1.3.0 # via jsonschema
183+
urllib3==2.4.0 # via requests
184+
virtualenv==20.31.2 # via pre-commit, tox
185+
wcwidth==0.2.13 # via prompt-toolkit
186+
webcolors==24.11.1 # via jsonschema
187+
webencodings==0.5.1 # via bleach, tinycss2
188+
websocket-client==1.8.0 # via jupyter-server
189+
wheel==0.45.1 # via -r requirements.txt
190+
widgetsnbextension==4.0.14 # via ipywidgets
191+
xxhash==3.5.0 # via datasets
192+
yarl==1.20.0 # via aiohttp
193+
setuptools==80.8.0 # via jupyterlab, triton

‎constraints-dev.txt.in

Whitespace-only changes.

0 commit comments

Comments
 (0)