Skip to content

Commit 5aec14b

Browse files
authored
Correct issues with testing in CI (#296)
Adds a patch to account for empty testing simulator sets. Also adds CI to confirm the testbed works in a CI environment.
1 parent 979bfaa commit 5aec14b

File tree

3 files changed

+92
-155
lines changed

3 files changed

+92
-155
lines changed

.github/workflows/ci.yaml

Lines changed: 81 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -90,16 +90,6 @@ jobs:
9090
fail-fast: false
9191
matrix:
9292
target: ['macOS', 'iOS', 'tvOS', 'watchOS']
93-
include:
94-
- briefcase-run-args:
95-
- run-tests: false
96-
97-
- target: macOS
98-
run-tests: true
99-
100-
- target: iOS
101-
briefcase-run-args: ' -d "iPhone SE (3rd generation)"'
102-
run-tests: true
10393

10494
steps:
10595
- uses: actions/[email protected]
@@ -125,20 +115,98 @@ jobs:
125115
name: Python-${{ needs.config.outputs.PYTHON_VER }}-${{ matrix.target }}-support.${{ needs.config.outputs.BUILD_NUMBER }}.tar.gz
126116
path: dist/Python-${{ needs.config.outputs.PYTHON_VER }}-${{ matrix.target }}-support.${{ needs.config.outputs.BUILD_NUMBER }}.tar.gz
127117

118+
briefcase-testbed:
119+
name: Briefcase testbed (${{ matrix.target }})
120+
runs-on: macOS-latest
121+
needs: [ config, build ]
122+
strategy:
123+
fail-fast: false
124+
matrix:
125+
target: ["macOS", "iOS"]
126+
include:
127+
- briefcase-run-args:
128+
129+
- target: iOS
130+
briefcase-run-args: ' -d "iPhone SE (3rd generation)"'
131+
132+
steps:
133+
- uses: actions/[email protected]
134+
135+
- name: Get build artifact
136+
uses: actions/[email protected]
137+
with:
138+
pattern: Python-${{ needs.config.outputs.PYTHON_VER }}-${{ matrix.target }}-support.${{ needs.config.outputs.BUILD_NUMBER }}.tar.gz
139+
path: dist
140+
merge-multiple: true
141+
142+
- name: Set up Python
143+
uses: actions/[email protected]
144+
with:
145+
# Appending -dev ensures that we can always build the dev release.
146+
# It's a no-op for versions that have been published.
147+
python-version: ${{ needs.config.outputs.PYTHON_VER }}-dev
148+
# Ensure that we *always* use the latest build, not a cached version.
149+
# It's an edge case, but when a new alpha is released, we need to use it ASAP.
150+
check-latest: true
151+
128152
- uses: actions/[email protected]
129-
if: matrix.run-tests
130153
with:
131154
repository: beeware/Python-support-testbed
132155
path: Python-support-testbed
133156

134157
- name: Install dependencies
135-
if: matrix.run-tests
136158
run: |
137159
# Use the development version of Briefcase
138160
python -m pip install git+https://github.com/beeware/briefcase.git
139161
140162
- name: Run support testbed check
141-
if: matrix.run-tests
142163
timeout-minutes: 10
143164
working-directory: Python-support-testbed
144165
run: briefcase run ${{ matrix.target }} Xcode --test ${{ matrix.briefcase-run-args }} -C support_package=\'../dist/Python-${{ needs.config.outputs.PYTHON_VER }}-${{ matrix.target }}-support.${{ needs.config.outputs.BUILD_NUMBER }}.tar.gz\'
166+
167+
cpython-testbed:
168+
name: CPython testbed (${{ matrix.target }})
169+
runs-on: macOS-latest
170+
needs: [ config, build ]
171+
strategy:
172+
fail-fast: false
173+
matrix:
174+
target: ["iOS"]
175+
176+
steps:
177+
- uses: actions/[email protected]
178+
179+
- name: Get build artifact
180+
uses: actions/[email protected]
181+
with:
182+
pattern: Python-${{ needs.config.outputs.PYTHON_VER }}-${{ matrix.target }}-support.${{ needs.config.outputs.BUILD_NUMBER }}.tar.gz
183+
path: dist
184+
merge-multiple: true
185+
186+
- name: Set up Python
187+
uses: actions/[email protected]
188+
with:
189+
# Appending -dev ensures that we can always build the dev release.
190+
# It's a no-op for versions that have been published.
191+
python-version: ${{ needs.config.outputs.PYTHON_VER }}-dev
192+
# Ensure that we *always* use the latest build, not a cached version.
193+
# It's an edge case, but when a new alpha is released, we need to use it ASAP.
194+
check-latest: true
195+
196+
- name: Unpack support package
197+
run: |
198+
mkdir support
199+
cd support
200+
tar zxvf ../dist/Python-${{ needs.config.outputs.PYTHON_VER }}-${{ matrix.target }}-support.${{ needs.config.outputs.BUILD_NUMBER }}.tar.gz
201+
202+
- name: Run CPython testbed
203+
timeout-minutes: 10
204+
working-directory: support
205+
run: |
206+
# Run a representative subset of CPython core tests:
207+
# - test_builtin as a test of core language tools
208+
# - test_grammar as a test of core language features
209+
# - test_os as a test of system library calls
210+
# - test_bz2 as a simple test of third party libraries
211+
# - test_ctypes as a test of FFI
212+
python -m testbed run -- test --single-process --rerun -W test_builtin test_grammar test_os test_bz2 test_ctypes

Makefile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ BUILD_NUMBER=custom
1919
# of a release cycle, as official binaries won't be published.
2020
# PYTHON_MICRO_VERSION is the full version number, without any alpha/beta/rc suffix. (e.g., 3.10.0)
2121
# PYTHON_VER is the major/minor version (e.g., 3.10)
22-
PYTHON_VERSION=3.13.3
22+
PYTHON_VERSION=3.13.4
2323
PYTHON_PKG_VERSION=$(PYTHON_VERSION)
2424
PYTHON_MICRO_VERSION=$(shell echo $(PYTHON_VERSION) | grep -Eo "\d+\.\d+\.\d+")
2525
PYTHON_PKG_MICRO_VERSION=$(shell echo $(PYTHON_PKG_VERSION) | grep -Eo "\d+\.\d+\.\d+")

patch/Python/Python.patch

Lines changed: 10 additions & 141 deletions
Original file line numberDiff line numberDiff line change
@@ -132,7 +132,7 @@ index 510c7b9568a..810b08879f6 100644
132132
import _osx_support
133133
osname, release, machine = _osx_support.get_platform_osx(
134134
diff --git a/Misc/platform_triplet.c b/Misc/platform_triplet.c
135-
index ec0857a4a99..2350e9dc821 100644
135+
index f5cd73bdea8..50f2f8988b7 100644
136136
--- a/Misc/platform_triplet.c
137137
+++ b/Misc/platform_triplet.c
138138
@@ -257,6 +257,26 @@
@@ -1269,150 +1269,19 @@ index c3e261ecd9e..26ef7a95de4 100644
12691269
<key>CFBundleSupportedPlatforms</key>
12701270
<array>
12711271
<string>iPhoneOS</string>
1272-
diff --git a/iOS/Resources/bin/arm64-apple-ios-clang b/iOS/Resources/bin/arm64-apple-ios-clang
1273-
index c39519cd1f8..f50d5b5142f 100755
1274-
--- a/iOS/Resources/bin/arm64-apple-ios-clang
1275-
+++ b/iOS/Resources/bin/arm64-apple-ios-clang
1276-
@@ -1,2 +1,2 @@
1277-
#!/bin/sh
1278-
-xcrun --sdk iphoneos${IOS_SDK_VERSION} clang -target arm64-apple-ios "$@"
1279-
+xcrun --sdk iphoneos${IOS_SDK_VERSION} clang -target arm64-apple-ios${IPHONEOS_DEPLOYMENT_TARGET} "$@"
1280-
diff --git a/iOS/Resources/bin/arm64-apple-ios-clang++ b/iOS/Resources/bin/arm64-apple-ios-clang++
1281-
index d9b12925f38..0794731d7dc 100755
1282-
--- a/iOS/Resources/bin/arm64-apple-ios-clang++
1283-
+++ b/iOS/Resources/bin/arm64-apple-ios-clang++
1284-
@@ -1,2 +1,2 @@
1285-
#!/bin/sh
1286-
-xcrun --sdk iphoneos${IOS_SDK_VERSION} clang++ -target arm64-apple-ios "$@"
1287-
+xcrun --sdk iphoneos${IOS_SDK_VERSION} clang++ -target arm64-apple-ios${IPHONEOS_DEPLOYMENT_TARGET} "$@"
1288-
diff --git a/iOS/Resources/bin/arm64-apple-ios-cpp b/iOS/Resources/bin/arm64-apple-ios-cpp
1289-
index 24da23d3448..24fa1506bab 100755
1290-
--- a/iOS/Resources/bin/arm64-apple-ios-cpp
1291-
+++ b/iOS/Resources/bin/arm64-apple-ios-cpp
1292-
@@ -1,2 +1,2 @@
1293-
#!/bin/sh
1294-
-xcrun --sdk iphoneos${IOS_SDK_VERSION} clang -target arm64-apple-ios -E "$@"
1295-
+xcrun --sdk iphoneos${IOS_SDK_VERSION} clang -target arm64-apple-ios${IPHONEOS_DEPLOYMENT_TARGET} -E "$@"
1296-
diff --git a/iOS/Resources/bin/arm64-apple-ios-simulator-clang b/iOS/Resources/bin/arm64-apple-ios-simulator-clang
1297-
index 92e8d853d6e..4891a00876e 100755
1298-
--- a/iOS/Resources/bin/arm64-apple-ios-simulator-clang
1299-
+++ b/iOS/Resources/bin/arm64-apple-ios-simulator-clang
1300-
@@ -1,2 +1,2 @@
1301-
#!/bin/sh
1302-
-xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang -target arm64-apple-ios-simulator "$@"
1303-
+xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang -target arm64-apple-ios${IPHONEOS_DEPLOYMENT_TARGET}-simulator "$@"
1304-
diff --git a/iOS/Resources/bin/arm64-apple-ios-simulator-clang++ b/iOS/Resources/bin/arm64-apple-ios-simulator-clang++
1305-
index 076469cc70c..58b2a5f6f18 100755
1306-
--- a/iOS/Resources/bin/arm64-apple-ios-simulator-clang++
1307-
+++ b/iOS/Resources/bin/arm64-apple-ios-simulator-clang++
1308-
@@ -1,2 +1,2 @@
1309-
#!/bin/sh
1310-
-xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang++ -target arm64-apple-ios-simulator "$@"
1311-
+xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang++ -target arm64-apple-ios${IPHONEOS_DEPLOYMENT_TARGET}-simulator "$@"
1312-
diff --git a/iOS/Resources/bin/arm64-apple-ios-simulator-cpp b/iOS/Resources/bin/arm64-apple-ios-simulator-cpp
1313-
index c57f28cee5b..c9df94e8b7c 100755
1314-
--- a/iOS/Resources/bin/arm64-apple-ios-simulator-cpp
1315-
+++ b/iOS/Resources/bin/arm64-apple-ios-simulator-cpp
1316-
@@ -1,2 +1,2 @@
1317-
#!/bin/sh
1318-
-xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang -target arm64-apple-ios-simulator -E "$@"
1319-
+xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang -target arm64-apple-ios${IPHONEOS_DEPLOYMENT_TARGET}-simulator -E "$@"
1320-
diff --git a/iOS/Resources/bin/x86_64-apple-ios-simulator-clang b/iOS/Resources/bin/x86_64-apple-ios-simulator-clang
1321-
index 17cbe0c8a1e..f4739a7b945 100755
1322-
--- a/iOS/Resources/bin/x86_64-apple-ios-simulator-clang
1323-
+++ b/iOS/Resources/bin/x86_64-apple-ios-simulator-clang
1324-
@@ -1,2 +1,2 @@
1325-
#!/bin/sh
1326-
-xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang -target x86_64-apple-ios-simulator "$@"
1327-
+xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang -target x86_64-apple-ios${IPHONEOS_DEPLOYMENT_TARGET}-simulator "$@"
1328-
diff --git a/iOS/Resources/bin/x86_64-apple-ios-simulator-clang++ b/iOS/Resources/bin/x86_64-apple-ios-simulator-clang++
1329-
index 565d47b24c2..c348ae4c103 100755
1330-
--- a/iOS/Resources/bin/x86_64-apple-ios-simulator-clang++
1331-
+++ b/iOS/Resources/bin/x86_64-apple-ios-simulator-clang++
1332-
@@ -1,2 +1,2 @@
1333-
#!/bin/sh
1334-
-xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang++ -target x86_64-apple-ios-simulator "$@"
1335-
+xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang++ -target x86_64-apple-ios${IPHONEOS_DEPLOYMENT_TARGET}-simulator "$@"
1336-
diff --git a/iOS/Resources/bin/x86_64-apple-ios-simulator-cpp b/iOS/Resources/bin/x86_64-apple-ios-simulator-cpp
1337-
index 63fc8e8de2d..6d7f8084c9f 100755
1338-
--- a/iOS/Resources/bin/x86_64-apple-ios-simulator-cpp
1339-
+++ b/iOS/Resources/bin/x86_64-apple-ios-simulator-cpp
1340-
@@ -1,2 +1,2 @@
1341-
#!/bin/sh
1342-
-xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang -target x86_64-apple-ios-simulator -E "$@"
1343-
+xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang -target x86_64-apple-ios${IPHONEOS_DEPLOYMENT_TARGET}-simulator -E "$@"
13441272
diff --git a/iOS/testbed/__main__.py b/iOS/testbed/__main__.py
1345-
index b436c9af99d..c05497ede3a 100644
1273+
index c05497ede3a..1146bf3b988 100644
13461274
--- a/iOS/testbed/__main__.py
13471275
+++ b/iOS/testbed/__main__.py
1348-
@@ -123,6 +123,36 @@
1349-
)
1350-
1351-
1352-
+# Select a simulator device to use.
1353-
+async def select_simulator_device():
1354-
+ # List the testing simulators, in JSON format
1355-
+ raw_json = await async_check_output(
1356-
+ "xcrun", "simctl", "--set", "testing", "list", "-j"
1357-
+ )
1358-
+ json_data = json.loads(raw_json)
1359-
+
1360-
+ # Any device will do; we'll look for "SE" devices - but the name isn't
1361-
+ # consistent over time. Older Xcode versions will use "iPhone SE (Nth
1362-
+ # generation)"; As of 2025, they've started using "iPhone 16e".
1363-
+ #
1364-
+ # When Xcode is updated after a new release, new devices will be available
1365-
+ # and old ones will be dropped from the set available on the latest iOS
1366-
+ # version. Select the one with the highest minimum runtime version - this
1367-
+ # is an indicator of the "newest" released device, which should always be
1368-
+ # supported on the "most recent" iOS version.
1369-
+ se_simulators = sorted(
1370-
+ (devicetype["minRuntimeVersion"], devicetype["name"])
1371-
+ for devicetype in json_data["devicetypes"]
1372-
+ if devicetype["productFamily"] == "iPhone"
1373-
+ and (
1374-
+ ("iPhone " in devicetype["name"] and devicetype["name"].endswith("e"))
1375-
+ or "iPhone SE " in devicetype["name"]
1376-
+ )
1377-
+ )
1378-
+
1379-
+ return se_simulators[-1][1]
1380-
+
1381-
+
1382-
# Return a list of UDIDs associated with booted simulators
1383-
async def list_devices():
1384-
try:
1385-
@@ -371,12 +401,16 @@
1386-
plistlib.dump(info, f)
1387-
1388-
1389-
-async def run_testbed(simulator: str, args: list[str], verbose: bool=False):
1390-
+async def run_testbed(simulator: str | None, args: list[str], verbose: bool=False):
1391-
location = Path(__file__).parent
1392-
print("Updating plist...", end="", flush=True)
1393-
update_plist(location, args)
1394-
print(" done.", flush=True)
1395-
1396-
+ if simulator is None:
1397-
+ simulator = await select_simulator_device()
1398-
+ print(f"Running test on {simulator}", flush=True)
1399-
+
1400-
# We need to get an exclusive lock on simulator creation, to avoid issues
1401-
# with multiple simulators starting and being unable to tell which
1402-
# simulator is due to which testbed instance. See
1403-
@@ -453,8 +487,10 @@
1404-
)
1405-
run.add_argument(
1406-
"--simulator",
1407-
- default="iPhone SE (3rd Generation)",
1408-
- help="The name of the simulator to use (default: 'iPhone SE (3rd Generation)')",
1409-
+ help=(
1410-
+ "The name of the simulator to use (eg: 'iPhone 16e'). Defaults to ",
1411-
+ "the most recently released 'entry level' iPhone device."
1412-
+ )
1276+
@@ -127,7 +127,7 @@
1277+
async def select_simulator_device():
1278+
# List the testing simulators, in JSON format
1279+
raw_json = await async_check_output(
1280+
- "xcrun", "simctl", "--set", "testing", "list", "-j"
1281+
+ "xcrun", "simctl", "list", "-j"
14131282
)
1414-
run.add_argument(
1415-
"-v", "--verbose",
1283+
json_data = json.loads(raw_json)
1284+
14161285
--- /dev/null
14171286
+++ b/tvOS/README.rst
14181287
@@ -0,0 +1,108 @@

0 commit comments

Comments
 (0)