Skip to content

Commit 3790024

Browse files
author
msftbot[bot]
authored
Split out Smoke Tests and Unit Tests into a separate Jobs (#3617)
## Fixes #3560 This PR does the following: - [x] Split our tests out of the single Job into two separates Jobs, these can hopefully run in parallel? We should also get separate status checks for each job? (Will see when we run this here.) - [x] Adds a UWP Baseline Blank App, this will be useful for #3600 to compare what the minimum app size is currently with no extra dependencies. - [x] Split out UI Tests as a separate step from the Unit Tests - [x] Add Application Metrics Report to Smoke Tests #3600 (could be a separate PR I suppose) ## PR Type - Build or CI related changes ## What is the current behavior? Everything is done in one big job. ## What is the new behavior? We have more granularity on our pipeline ## PR Checklist Please check if your PR fulfills the following requirements: - [ ] Tested code with current [supported SDKs](../readme.md#supported) - [ ] Pull Request has been submitted to the documentation repository [instructions](..\contributing.md#docs). Link: <!-- docs PR link --> - [ ] Sample in sample app has been added / updated (for bug fixes / features) - [ ] Icon has been created (if new sample) following the [Thumbnail Style Guide and templates](https://github.com/windows-toolkit/WindowsCommunityToolkit-design-assets) - [ ] Tests for the changes have been added (for bug fixes / features) (if applicable) - [ ] Header has been added to all new source files (run *build/UpdateHeaders.bat*) - [ ] Contains **NO** breaking changes <!-- If this PR contains a breaking change, please describe the impact and migration path for existing applications below. Please note that breaking changes are likely to be rejected within minor release cycles or held until major versions. --> ## Other information
2 parents 8ccfaff + ab1e74a commit 3790024

File tree

7 files changed

+230
-26
lines changed

7 files changed

+230
-26
lines changed

‎SmokeTests/SmokeTest.csproj

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -112,18 +112,18 @@
112112
<Version>2.4.3</Version>
113113
</PackageReference>
114114
</ItemGroup>
115-
<ItemGroup Condition="'$(CurrentProject)' != ''">
115+
<ItemGroup Condition="'$(CurrentProject)' != '' and '$(CurrentProject)' != 'UWPBaseline'">
116116
<PackageReference Include="$(CurrentProject)" Version="7.*-*" />
117117
</ItemGroup>
118118
<PropertyGroup Condition=" '$(VisualStudioVersion)' == '' or '$(VisualStudioVersion)' &lt; '14.0' ">
119119
<VisualStudioVersion>14.0</VisualStudioVersion>
120120
</PropertyGroup>
121121
<Import Project="$(MSBuildExtensionsPath)\Microsoft\WindowsXaml\v$(VisualStudioVersion)\Microsoft.Windows.UI.Xaml.CSharp.targets" />
122122
<Target Name="BeforeBuild">
123-
<ItemGroup>
123+
<ItemGroup Condition="$(CurrentProject) != 'UWPBaseline'">
124124
<ToolkitNugets Include="../bin/nupkg/$(CurrentProject).*.nupkg"/>
125125
<ToolkitNuget Condition="$([System.Text.RegularExpressions.Regex]::IsMatch('%(Identity)', `$(CurrentProject).([0-9]+)\.([0-9]+)\.([0-9]+)(?:-([0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*))?(?:\+[0-9A-Za-z-]+)?.nupkg`))" Include="@(ToolkitNugets)"/>
126126
</ItemGroup>
127-
<Error Condition="'@(ToolkitNuget)' == ''" Text="NuGet $(CurrentProject).[SEMVER].nupkg doesn't exist!"/>
127+
<Error Condition="'@(ToolkitNuget)' == '' and $(CurrentProject) != 'UWPBaseline'" Text="NuGet $(CurrentProject).[SEMVER].nupkg doesn't exist!"/>
128128
</Target>
129129
</Project>

‎SmokeTests/SmokeTestAnalysis.ps1

Lines changed: 149 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,149 @@
1+
$ErrorActionPreference = "Stop"
2+
$ProgressPreference="SilentlyContinue"
3+
4+
Write-Host "Running Smoke Test Package Analyis"
5+
Write-Host "----------------------------------"
6+
7+
# Our script is at our SmokeTest root, we want to look for the AppPackages folder
8+
$PackagePath = $PSScriptRoot + "\AppPackages\"
9+
$FilePattern = "SmokeTest_{0}_x86_bundle.msixupload"
10+
$BaselineName = $FilePattern -f "UWPBaseline"
11+
$TempFolder = "ExplodedArchive"
12+
13+
function Expand-MsixUploadPackage {
14+
param (
15+
[string]$PackageFile,
16+
[string]$Destination
17+
)
18+
19+
$ZipUpload = $PackageFile.Replace("msixupload", "zip")
20+
21+
Move-Item $PackageFile -Destination $ZipUpload
22+
23+
Expand-Archive $ZipUpload -DestinationPath $Destination
24+
25+
Move-Item $ZipUpload -Destination $PackageFile
26+
27+
Push-Location $Destination
28+
29+
$Bundle = (Get-ChildItem "*.msixbundle").Name
30+
$ZipBundle = $Bundle.Replace("msixbundle", "zip")
31+
32+
Move-Item $Bundle -Destination $ZipBundle
33+
34+
Expand-Archive $ZipBundle -DestinationPath .
35+
36+
Remove-Item $ZipBundle
37+
38+
$msix = (Get-ChildItem "*.msix").Name
39+
$ZipMSIX = $msix.Replace("msix", "zip")
40+
41+
Move-Item $msix -Destination $ZipMSIX
42+
43+
Expand-Archive $ZipMSIX -DestinationPath . -Force # Force here as we have some duplicate file names we don't really care about from parent archives
44+
45+
Remove-Item $ZipMSIX
46+
47+
Pop-Location
48+
}
49+
50+
if (Test-Path $PackagePath)
51+
{
52+
Push-Location $PackagePath
53+
54+
Write-Host "Extracting Baseline..."
55+
56+
# TODO: Theoretically we could grab bits from the bin directory instead of having to expand each package, not sure about what we ignore though
57+
Expand-MsixUploadPackage $BaselineName -Destination $TempFolder
58+
59+
# Get all the base file info only (grab stuff in directories but not the directories themselves)
60+
$BaselineFiles = Get-ChildItem $TempFolder -Recurse -Attributes !Directory -Exclude "SmokeTest*"
61+
$SmokeTestFiles = Get-ChildItem $TempFolder -Recurse -Attributes !Directory -Include "SmokeTest*"
62+
63+
$BaselineFootprint = ($BaselineFiles | Measure-Object -Property Length -sum).Sum + ($SmokeTestFiles | Measure-Object -Property Length -sum).Sum
64+
Write-Host ("Baseline Footprint: {0:n0} bytes" -f $BaselineFootprint)
65+
Write-Host "-----------------------------------------"
66+
67+
$PackageList = Get-ChildItem "$PackagePath*.msixupload" -Exclude $BaselineName
68+
69+
#$i = 0
70+
foreach ($Package in $PackageList)
71+
{
72+
#Write-Progress -Id 0 -Activity "Comparing Against Baseline..." -Status "Prepping Package" -PercentComplete (($i++ / $PackageList.count)*100) -CurrentOperation $Package.Name
73+
74+
# Make sure we've cleaned-up the last archive
75+
Remove-Item $TempFolder -Recurse -Force
76+
77+
#$ProgressPreference="SilentlyContinue"
78+
Expand-MsixUploadPackage $Package.Name -Destination $TempFolder
79+
#$ProgressPreference="Continue"
80+
81+
[System.Collections.ArrayList]$PackageFiles = Get-ChildItem $TempFolder -Recurse -Attributes !Directory -Exclude "SmokeTest*"
82+
$PackageSmokeTestFiles = Get-ChildItem $TempFolder -Recurse -Attributes !Directory -Include "SmokeTest*"
83+
84+
# TODO: Make function or regex better to extra package name more easily based on a template string at the top or something...
85+
$PackageShortName = $Package.Name.substring(10, $Package.Name.Length - 32)
86+
Write-Host ("{0} Additional Footprint: {1:n0} bytes" -f $PackageShortName, (($PackageFiles | Measure-Object -Property Length -sum).Sum + ($PackageSmokeTestFiles | Measure-Object -Property Length -sum).Sum - $BaselineFootprint))
87+
88+
# Quick check on the base exe file/symbols differences
89+
foreach ($file in $SmokeTestFiles)
90+
{
91+
$match = $null
92+
$match = $PackageSmokeTestFiles | Where-Object {$_.Extension -eq $file.Extension}
93+
if ($null -ne $match)
94+
{
95+
Write-Host (" App Diff: ({0}) = {1:n0}" -f $file.Extension, ($match.Length - $file.Length)) -ForegroundColor DarkCyan
96+
}
97+
}
98+
99+
#$j = 0
100+
foreach ($file in $BaselineFiles)
101+
{
102+
#Write-Progress -Id 1 -ParentId 0 -Activity "Comparing Against Baseline..." -Status "Comparing Package" -PercentComplete (($j++ / $BaselineFiles.count)*100) -CurrentOperation $file.Name
103+
104+
$match = $null
105+
$match = $PackageFiles | Where-Object {$_.Name -eq $file.Name}
106+
if ($null -ne $match)
107+
{
108+
# File was in baseline, but has a different size
109+
if ($match.Length -ne $file.Length)
110+
{
111+
Write-Host (" Size Diff: {0} = {1:n0}" -f $file.Name, ($match.Length - $file.Length)) -ForegroundColor Magenta
112+
}
113+
114+
# Remove checked files (performance) and also remaining are new
115+
$PackageFiles.Remove($match)
116+
}
117+
}
118+
119+
# List remaining (new) files to this package
120+
foreach ($file in $PackageFiles)
121+
{
122+
if ($file.Name -match $PackageShortName)
123+
{
124+
Write-Host (" Lib (self): {0} = {1:n0}" -f $file.Name, $file.Length) -ForegroundColor White
125+
}
126+
else
127+
{
128+
Write-Host (" Additional: {0} = {1:n0}" -f $file.Name, $file.Length) -ForegroundColor Yellow
129+
}
130+
}
131+
132+
# TODO: Especially if we add comparison to the main branch, we should format as an actual table and colorize via VT: https://stackoverflow.com/a/49038815/8798708
133+
134+
#Write-Progress -Id 1 -ParentId 0 -Activity "Comparing Against Baseline..." -Completed
135+
Write-Host "-----------------------------------------"
136+
Write-Host
137+
}
138+
139+
#Write-Progress -Id 0 -Activity "Comparing Against Baseline..." -Completed
140+
141+
# Clean-up
142+
Remove-Item $TempFolder -Recurse -Force
143+
144+
Pop-Location
145+
}
146+
else
147+
{
148+
Write-Error "Path $PackagePath not found for analysis!"
149+
}

‎SmokeTests/SmokeTests.proj

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,15 +4,15 @@
44
<PropertyGroup>
55
<BuildPlatforms>x86</BuildPlatforms>
66
<BuildConfigurations>Release</BuildConfigurations>
7-
<ToolkitPackages>Microsoft.Toolkit;Microsoft.Toolkit.HighPerformance;Microsoft.Toolkit.Parsers;Microsoft.Toolkit.Mvvm;Microsoft.Toolkit.Services;Microsoft.Toolkit.Uwp;Microsoft.Toolkit.Uwp.Connectivity;Microsoft.Toolkit.Uwp.DeveloperTools;Microsoft.Toolkit.Uwp.Input.GazeInteraction;Microsoft.Toolkit.Uwp.Notifications;Microsoft.Toolkit.Uwp.UI;Microsoft.Toolkit.Uwp.UI.Animations;Microsoft.Toolkit.Uwp.UI.Controls;Microsoft.Toolkit.Uwp.UI.Controls.DataGrid;Microsoft.Toolkit.Uwp.UI.Controls.Layout;Microsoft.Toolkit.Uwp.UI.Media;Microsoft.Toolkit.Uwp.UI.Controls.Markdown</ToolkitPackages>
7+
<ToolkitPackages>UWPBaseline;Microsoft.Toolkit;Microsoft.Toolkit.HighPerformance;Microsoft.Toolkit.Parsers;Microsoft.Toolkit.Mvvm;Microsoft.Toolkit.Services;Microsoft.Toolkit.Uwp;Microsoft.Toolkit.Uwp.Connectivity;Microsoft.Toolkit.Uwp.DeveloperTools;Microsoft.Toolkit.Uwp.Input.GazeInteraction;Microsoft.Toolkit.Uwp.Notifications;Microsoft.Toolkit.Uwp.UI;Microsoft.Toolkit.Uwp.UI.Animations;Microsoft.Toolkit.Uwp.UI.Controls;Microsoft.Toolkit.Uwp.UI.Controls.DataGrid;Microsoft.Toolkit.Uwp.UI.Controls.Layout;Microsoft.Toolkit.Uwp.UI.Media;Microsoft.Toolkit.Uwp.UI.Controls.Markdown</ToolkitPackages>
88
</PropertyGroup>
99

1010
<Target Name="Build"
1111
DependsOnTargets="ChooseProjectsToBuild"
1212
Inputs="@(ProjectsToBuild)"
1313
Outputs="%(Filename)">
1414

15-
<Message Importance="High" Text="Building project %(ProjectsToBuild.Filename): (%(ProjectsToBuild.Configuration)|%(ProjectsToBuild.Platform))" />
15+
<Message Importance="High" Text="Building project %(ProjectsToBuild.Identity): (%(ProjectsToBuild.Configuration)|%(ProjectsToBuild.Platform))" />
1616

1717
<MSBuild Projects="SmokeTest.csproj"
1818
Targets="restore;build"

‎SmokeTests/UWPBaseline/MainPage.xaml

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
<Page
2+
x:Class="SmokeTest.MainPage"
3+
xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
4+
xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
5+
xmlns:local="using:SmokeTest"
6+
xmlns:d="http://schemas.microsoft.com/expression/blend/2008"
7+
xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006"
8+
mc:Ignorable="d"
9+
Background="{ThemeResource ApplicationPageBackgroundThemeBrush}">
10+
11+
<Grid/>
12+
</Page>

‎SmokeTests/UWPBaseline/MainPage.xaml.cs

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
// Licensed to the .NET Foundation under one or more agreements.
2+
// The .NET Foundation licenses this file to you under the MIT license.
3+
// See the LICENSE file in the project root for more information.
4+
5+
namespace SmokeTest
6+
{
7+
public sealed partial class MainPage
8+
{
9+
public MainPage()
10+
{
11+
InitializeComponent();
12+
}
13+
}
14+
}

‎azure-pipelines.yml

Lines changed: 43 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -13,8 +13,9 @@ variables:
1313
BuildConfiguration: Release
1414

1515
jobs:
16-
- job: ToolkitBuild
17-
timeoutInMinutes: 120
16+
### BUILD ###
17+
- job: BuildBits
18+
timeoutInMinutes: 60
1819

1920
steps:
2021
- task: BatchScript@1
@@ -45,15 +46,10 @@ jobs:
4546
- powershell: .\build\build.ps1 -target=Build
4647
displayName: Build
4748

49+
### Unit Tests ###
50+
4851
- powershell: .\build\build.ps1 -target=Test
4952
displayName: Test
50-
timeoutInMinutes: 15
51-
52-
- powershell: .\build\build.ps1 -target=Package
53-
displayName: Package
54-
55-
- powershell: .\build\build.ps1 -target=SmokeTest
56-
displayName: SmokeTest
5753

5854
- task: PublishTestResults@2
5955
inputs:
@@ -62,19 +58,29 @@ jobs:
6258
displayName: Publish Test Results
6359
condition: always()
6460

61+
- task: PublishPipelineArtifact@1
62+
displayName: Publish Test WexLogFileOutput
63+
inputs:
64+
targetPath: .\build\WexLogFileOutput
65+
artifactName: WexUnitTestErrorLogFileOutput
66+
condition: failed()
67+
68+
### UI Integration Tests ###
69+
70+
- powershell: .\build\build.ps1 -target=UITest
71+
displayName: UI Integration Tests
72+
6573
- task: PublishPipelineArtifact@1
6674
displayName: Publish UI Test Results
6775
inputs:
6876
targetPath: .\build\UITestResults.wtl
69-
artifactName: WexLogFileOutput
77+
artifactName: WexUITestLogFileOutput
7078
condition: always()
7179

72-
- task: PublishPipelineArtifact@1
73-
displayName: Publish Test WexLogFileOutput
74-
inputs:
75-
targetPath: .\build\WexLogFileOutput
76-
artifactName: WexErrorLogFileOutput
77-
condition: failed()
80+
### Package ###
81+
82+
- powershell: .\build\build.ps1 -target=Package
83+
displayName: Package
7884

7985
- task: PowerShell@2
8086
displayName: Authenticode Sign Packages
@@ -86,13 +92,28 @@ jobs:
8692
ArtifactDirectory: bin\nupkg
8793
condition: and(succeeded(), not(eq(variables['build.reason'], 'PullRequest')), not(eq(variables['SignClientSecret'], '')), not(eq(variables['SignClientUser'], '')))
8894

89-
- task: PublishBuildArtifacts@1
95+
- task: PublishPipelineArtifact@1
9096
displayName: Publish Package Artifacts
9197
inputs:
92-
pathToPublish: .\bin\nupkg
93-
artifactType: container
98+
targetPath: .\bin\nupkg
9499
artifactName: Packages
95100

101+
### Smoke Tests ###
102+
103+
- job: SmokeTests
104+
dependsOn: BuildBits
105+
timeoutInMinutes: 40
106+
107+
steps:
108+
- task: DownloadPipelineArtifact@2
109+
displayName: Download NuGet Packages Artifact
110+
inputs:
111+
artifact: Packages
112+
path: .\bin\nupkg
113+
114+
- powershell: .\build\build.ps1 -target=SmokeTest
115+
displayName: SmokeTest
116+
96117
- task: CopyFiles@2
97118
inputs:
98119
sourceFolder: .\SmokeTests\AppPackages
@@ -105,3 +126,6 @@ jobs:
105126
pathToPublish: $(build.artifactstagingdirectory)\SmokeTestBundles
106127
artifactType: container
107128
artifactName: SmokeTestBundles
129+
130+
- powershell: .\SmokeTests\SmokeTestAnalysis.ps1
131+
displayName: Analyze Package Sizes

‎build/build.cake

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -243,7 +243,7 @@ public string getMSTestAdapterPath(){
243243
}
244244

245245
Task("Test")
246-
.Description("Runs all Tests")
246+
.Description("Runs all Unit Tests")
247247
.Does(() =>
248248
{
249249
Information("\nRunning Unit Tests");
@@ -272,7 +272,11 @@ Task("Test")
272272
ArgumentCustomization = arg => arg.Append($"-s {baseDir}/.runsettings"),
273273
};
274274
DotNetCoreTest(file.FullPath, testSettings);
275-
}).DoesForEach(GetFiles(taefBinDir + "/**/UITests.Tests.TAEF.dll"), (file) =>
275+
}).DeferOnError();
276+
277+
Task("UITest")
278+
.Description("Runs all UI Tests")
279+
.DoesForEach(GetFiles(taefBinDir + "/**/UITests.Tests.TAEF.dll"), (file) =>
276280
{
277281
Information("\nRunning TAEF Interaction Tests");
278282

@@ -315,6 +319,7 @@ Task("MSTestUITest")
315319
Task("Default")
316320
.IsDependentOn("Build")
317321
.IsDependentOn("Test")
322+
.IsDependentOn("UITest")
318323
.IsDependentOn("Package");
319324

320325
Task("UpdateHeaders")

0 commit comments

Comments
 (0)