Skip to content

Commit 6fea9e5

Browse files
[CI] fix bitblas timeout (#445)
* fix bitblas timeout * fix wrong file * alyways check cancelled * Update unit_tests.yml
1 parent 6113128 commit 6fea9e5

File tree

1 file changed

+20
-15
lines changed

1 file changed

+20
-15
lines changed

.github/workflows/unit_tests.yml

Lines changed: 20 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,7 @@ jobs:
7676
build-zen4:
7777
runs-on: [self-hosted, zen4]
7878
needs: check-vm
79-
if: needs.check-vm.outputs.tag == 'zen4' && github.event.inputs.artifact_id == ''
79+
if: needs.check-vm.outputs.tag == 'zen4' && github.event.inputs.artifact_id == '' && !cancelled()
8080
container:
8181
image: ${{ needs.check-vm.outputs.ip }}:5000/modelcloud/gptqmodel:github-ci-v2
8282
steps:
@@ -179,7 +179,6 @@ jobs:
179179
- check-vm
180180
runs-on: self-hosted
181181
if: always() && !cancelled() && (needs.build-zen4.result == 'success' || github.event.inputs.artifact_id != '')
182-
timeout-minutes: 40
183182
container:
184183
image: ${{ needs.check-vm.outputs.ip }}:5000/modelcloud/gptqmodel:github-ci-v2
185184
strategy:
@@ -207,6 +206,10 @@ jobs:
207206
fi
208207
echo "RUN_ID=$run_id" >> $GITHUB_ENV
209208
echo "RUN_ID=${run_id}"
209+
210+
if grep -q "bitblas" tests/${{ matrix.test_script }}.py; then
211+
echo "BITBLAS=1" >> $GITHUB_ENV
212+
fi
210213
211214
- name: Checkout Codes
212215
uses: actions/checkout@v4
@@ -230,7 +233,7 @@ jobs:
230233
fi
231234
232235
- name: Download artifact
233-
if: env.DOWNLOADED == ''
236+
if: env.DOWNLOADED == '' && !cancelled()
234237
uses: actions/download-artifact@v4
235238
with:
236239
name: dist
@@ -239,9 +242,10 @@ jobs:
239242

240243
- name: Install wheel
241244
run: |
242-
uv pip install auto_round optimum bitblas==0.0.1.dev13 parameterized uvicorn -i http://${{ needs.check-vm.outputs.ip }}/simple/ --trusted-host ${{ needs.check-vm.outputs.ip }}
245+
uv pip install auto_round optimum bitblas==0.0.1.dev13 parameterized intel_extension_for_transformers uvicorn -i http://${{ needs.check-vm.outputs.ip }}/simple/ --trusted-host ${{ needs.check-vm.outputs.ip }}
243246
uv pip install dist/*.whl
244-
uv pip install torch -U
247+
bash -c "$(curl -L http://10.0.14.199/scripts/compiler/init_env.sh)" @ 12.4 2.4.1 3.11
248+
uv pip install transformers -U
245249
246250
- name: Check platform
247251
run: |
@@ -251,7 +255,6 @@ jobs:
251255
pip show torch
252256
echo "-----------"
253257
nvcc --version
254-
rm -rf ~/.cache/bitblas/nvidia/geforce-rtx-4090
255258
256259
- name: Find suitable GPU
257260
run: |
@@ -273,14 +276,16 @@ jobs:
273276
echo "STEP_TIMESTAMP=$timestamp" >> $GITHUB_ENV
274277
echo "CUDA_VISIBLE_DEVICES set to $gpu_id, timestamp=$timestamp"
275278
276-
- name: Install requirements
277-
run: |
278-
bash -c "$(curl -L http://10.0.14.199/scripts/compiler/init_env.sh)" @ 12.4 2.4.1 3.11
279-
uv pip install transformers -U
280-
281279
- name: Run tests
282-
if: ${{ !github.event.inputs.test_names || contains(github.event.inputs.test_names, matrix.test_script) }}
283-
run: pytest --durations=0 tests/${{ matrix.test_script }}.py
280+
timeout-minutes: ${{ env.BITBLAS == '1' && 120 || 40 }}
281+
if: ${{ !github.event.inputs.test_names || contains(github.event.inputs.test_names, matrix.test_script) }} && !cancelled()
282+
run: pytest --durations=0 tests/${{ matrix.test_script }}.py || { echo "ERROR=1" >> $GITHUB_ENV; exit 1; }
283+
284+
- name: Clear cache
285+
if: failure() && env.BITBLAS == '1' && env.ERROR == '1'
286+
run: |
287+
rm -rf ~/.cache/bitblas/nvidia/geforce-rtx-4090
288+
echo "clear bitblas cache"
284289
285290
- name: Release GPU
286291
if: always()
@@ -333,7 +338,7 @@ jobs:
333338
fi
334339
335340
- name: Download artifact
336-
if: env.DOWNLOADED == ''
341+
if: env.DOWNLOADED == '' && !cancelled()
337342
uses: actions/download-artifact@v4
338343
with:
339344
name: dist
@@ -344,5 +349,5 @@ jobs:
344349
run: uv pip install intel_extension_for_transformers dist/*.whl -i http://${{ needs.check-vm.outputs.ip }}/simple/ --trusted-host ${{ needs.check-vm.outputs.ip }}
345350

346351
- name: Run tests
347-
if: ${{ !github.event.inputs.test_names || contains(github.event.inputs.test_names, matrix.test_script) }}
352+
if: ${{ !github.event.inputs.test_names || contains(github.event.inputs.test_names, matrix.test_script) }} && !cancelled()
348353
run: pytest --durations=0 tests/${{ matrix.test_script }}.py

0 commit comments

Comments
 (0)