V16 QA Fixing the failing E2E tests (#19318)

* Updated nightly E2E pipeline

* Fixed failing E2E tests

* Skipped content tests wirh list view content due to an issue

* Updated tests due to UI changes

* Bumped version

* Added more waits

Author: nhudinh0309

build/nightly-E2E-test-pipelines.yml

@@ -10,7 +10,7 @@ schedules:
       include:
         - v14/dev
         - v15/dev
-        - v16/dev
+        - main
 
 variables:
   nodeVersion: 20
@@ -33,25 +33,17 @@ stages:
       - job: A
         displayName: Build Umbraco CMS
         pool:
-          vmImage: 'ubuntu-latest'
+          vmImage: "windows-latest"
         steps:
           - checkout: self
-            fetchDepth: 0
-            submodules: true
+            submodules: false
+            lfs: false,
+            fetchDepth: 500
+          - template: templates/backoffice-install.yml
           - task: UseDotNet@2
             displayName: Use .NET SDK from global.json
             inputs:
               useGlobalJson: true
-          - template: templates/backoffice-install.yml
-          - script: npm run build:for:cms
-            displayName: Run build (Bellissima)
-            workingDirectory: src/Umbraco.Web.UI.Client
-          - script: npm ci --no-fund --no-audit --prefer-offline
-            displayName: Run npm ci (Login)
-            workingDirectory: src/Umbraco.Web.UI.Login
-          - script: npm run build
-            displayName: Run npm build (Login)
-            workingDirectory: src/Umbraco.Web.UI.Login
           - task: DotNetCoreCLI@2
             displayName: Run dotnet restore
             inputs:
@@ -63,7 +55,7 @@ stages:
             inputs:
               command: build
               projects: $(solution)
-              arguments: '--configuration $(buildConfiguration) --no-restore --property:ContinuousIntegrationBuild=true --property:GeneratePackageOnBuild=true --property:PackageOutputPath=$(Build.ArtifactStagingDirectory)/nupkg'
+              arguments: "--configuration $(buildConfiguration) --no-restore --property:ContinuousIntegrationBuild=true --property:GeneratePackageOnBuild=true --property:PackageOutputPath=$(Build.ArtifactStagingDirectory)/nupkg"
           - task: PublishPipelineArtifact@1
             displayName: Publish nupkg
             inputs:
@@ -75,6 +67,33 @@ stages:
               targetPath: $(Build.SourcesDirectory)
               artifactName: build_output
 
+      - job: B
+        displayName: Build Bellissima Package
+        pool:
+          vmImage: "ubuntu-latest"
+        steps:
+          - checkout: self
+            submodules: false
+            lfs: false,
+            fetchDepth: 500
+          - template: templates/backoffice-install.yml
+          - script: npm run build:for:npm
+            displayName: Run build:for:npm
+            workingDirectory: src/Umbraco.Web.UI.Client
+          - bash: |
+              echo "##[command]Running npm pack"
+              echo "##[debug]Output directory: $(Build.ArtifactStagingDirectory)"
+              mkdir $(Build.ArtifactStagingDirectory)/npm
+              npm pack --pack-destination $(Build.ArtifactStagingDirectory)/npm
+              mv .npmrc $(Build.ArtifactStagingDirectory)/npm/
+            displayName: Run npm pack
+            workingDirectory: src/Umbraco.Web.UI.Client
+          - task: PublishPipelineArtifact@1
+            displayName: Publish Bellissima npm artifact
+            inputs:
+              targetPath: $(Build.ArtifactStagingDirectory)/npm
+              artifactName: npm
+
   - stage: E2E
     displayName: E2E Tests
     dependsOn: Build
@@ -209,9 +228,9 @@ stages:
             displayName: Wait for application
             workingDirectory: tests/Umbraco.Tests.AcceptanceTest
 
-          # Install Playwright and dependencies
-          - pwsh: npx playwright install --with-deps
-            displayName: Install Playwright
+            # Install Playwright and dependencies
+          - pwsh: npx playwright install chromium
+            displayName: Install Playwright only with Chromium browser
             workingDirectory: tests/Umbraco.Tests.AcceptanceTest
 
           # Test
@@ -382,8 +401,8 @@ stages:
             workingDirectory: tests/Umbraco.Tests.AcceptanceTest
 
           # Install Playwright and dependencies
-          - pwsh: npx playwright install --with-deps
-            displayName: Install Playwright
+          - pwsh: npx playwright install chromium
+            displayName: Install Playwright only with Chromium browser
             workingDirectory: tests/Umbraco.Tests.AcceptanceTest
 
           # Test
@@ -438,4 +457,4 @@ stages:
               testResultsFormat: 'JUnit'
               testResultsFiles: '*.xml'
               searchFolder: "tests/Umbraco.Tests.AcceptanceTest/results"
-              testRunTitle: "$(Agent.JobName)"
+              testRunTitle: "$(Agent.JobName)"

tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Content/BlockGrid/VariantBlockGrid.spec.ts

@@ -57,7 +57,6 @@ test('invariant document type with invariant block grid with invariant block wit
 
   // Assert
   //await umbracoUi.content.doesSuccessNotificationHaveText(NotificationConstantHelper.success.saved);
-  await umbracoUi.content.isErrorNotificationVisible(false);
   await umbracoUi.content.doesSuccessNotificationHaveText(NotificationConstantHelper.success.published);
 
   await umbracoUi.reloadPage();
@@ -79,8 +78,6 @@ test('can not create unsupported invariant document type with invariant block gr
   await umbracoUi.content.clickSaveAndPublishButton();
 
   // Assert
-  //await umbracoUi.content.doesSuccessNotificationHaveText(NotificationConstantHelper.success.saved);
-  await umbracoUi.content.isErrorNotificationVisible(false);
   await umbracoUi.content.doesErrorNotificationHaveText(NotificationConstantHelper.error.documentCouldNotBePublished);
 });
 
@@ -99,8 +96,6 @@ test('can not create unsupported invariant document type with invariant block gr
   await umbracoUi.content.clickSaveAndPublishButton();
 
   // Assert
-  //await umbracoUi.content.doesSuccessNotificationHaveText(NotificationConstantHelper.success.saved);
-  await umbracoUi.content.isErrorNotificationVisible(false);
   await umbracoUi.content.doesErrorNotificationHaveText(NotificationConstantHelper.error.documentCouldNotBePublished);
 });
 

tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Content/BlockList/VariantBlockList.spec.ts

@@ -79,8 +79,6 @@ test('can not create unsupported invariant document type with invariant block li
   await umbracoUi.content.clickSaveAndPublishButton();
 
   // Assert
-  //await umbracoUi.content.doesSuccessNotificationHaveText(NotificationConstantHelper.success.saved);
-  await umbracoUi.content.isErrorNotificationVisible(false);
   await umbracoUi.content.doesErrorNotificationHaveText(NotificationConstantHelper.error.documentCouldNotBePublished);
 });
 
@@ -98,8 +96,6 @@ test('can not create unsupported invariant document type with invariant block li
   await umbracoUi.content.clickSaveAndPublishButton();
 
   // Assert
-  //await umbracoUi.content.doesSuccessNotificationHaveText(NotificationConstantHelper.success.saved);
-  await umbracoUi.content.isErrorNotificationVisible(false);
   await umbracoUi.content.doesErrorNotificationHaveText(NotificationConstantHelper.error.documentCouldNotBePublished);
 });
 

tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Content/Clipboard/ClipboardBlockGridBlocks.spec.ts

@@ -267,7 +267,7 @@ test('can replace multiple blocks',  async ({umbracoApi, umbracoUi}) => {
   await umbracoUi.content.clickPasteButton();
   await umbracoUi.content.goToBlockGridBlockWithName(groupName, blockGridDataTypeName, elementTypeName, 0);
   await umbracoUi.content.doesBlockEditorBlockWithNameContainValue(elementGroupName, elementPropertyName, ConstantHelper.inputTypes.tipTap, blockPropertyValue);
-  await umbracoUi.content.clickCreateButton();
+  await umbracoUi.content.clickCreateModalButton();
   await umbracoUi.content.clickSaveButton();
 
   // Assert

tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Content/Clipboard/ClipboardBlockListBlocks.spec.ts

@@ -267,7 +267,7 @@ test('can replace multiple blocks', async ({umbracoApi, umbracoUi}) => {
   await umbracoUi.content.clickPasteButton();
   await umbracoUi.content.goToBlockListBlockWithName(groupName, blockListDataTypeName, elementTypeName, 0);
   await umbracoUi.content.doesBlockEditorBlockWithNameContainValue(elementGroupName, elementPropertyName, ConstantHelper.inputTypes.tipTap, blockPropertyValue);
-  await umbracoUi.content.clickCreateButton();
+  await umbracoUi.content.clickCreateModalButton();
   await umbracoUi.content.clickSaveButton();
 
   // Assert

tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Content/ContentInfoTab.spec.ts

@@ -47,7 +47,8 @@ test('can see correct information when published', async ({umbracoApi, umbracoUi
     hour12: true,
   });
   await umbracoUi.content.doesCreatedDateHaveText(expectedCreatedDate);
-  await umbracoUi.content.doesDocumentHaveLink(contentData.urls[0].url ? contentData.urls[0].url : '/');
+  const contentUrl = await umbracoApi.document.getDocumentUrl(contentId);
+  await umbracoUi.content.doesDocumentHaveLink(contentUrl);
   // TODO: Uncomment this when front-end is ready. Currently the publication status of content is not changed to "Published" immediately after publishing it
   //await umbracoUi.content.doesPublicationStatusHaveText(contentData.variants[0].state === 'Draft' ? 'Unpublished' : contentData.variants[0].state);
 });

tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Content/ContentWithCheckboxList.spec.ts

@@ -99,17 +99,13 @@ test('can not publish a mandatory checkbox list with an empty value', async ({um
   // Do not select any checkbox list values and the validation error appears
   await umbracoUi.content.clickSaveAndPublishButton();
   await umbracoUi.content.isValidationMessageVisible(ConstantHelper.validationMessages.emptyValue);
-  //await umbracoUi.content.doesSuccessNotificationHaveText(NotificationConstantHelper.success.saved);
-  await umbracoUi.content.isErrorNotificationVisible(false);
   await umbracoUi.content.doesErrorNotificationHaveText(NotificationConstantHelper.error.documentCouldNotBePublished);
   // Select a checkbox list value and the validation error disappears
   await umbracoUi.content.chooseCheckboxListOption(optionValues[0]);
   await umbracoUi.content.isValidationMessageVisible(ConstantHelper.validationMessages.emptyValue, false);
   await umbracoUi.content.clickSaveAndPublishButton();
 
   // Assert
-  //await umbracoUi.content.doesSuccessNotificationHaveText(NotificationConstantHelper.success.saved);
-  await umbracoUi.content.isErrorNotificationVisible(false);
   await umbracoUi.content.doesSuccessNotificationHaveText(NotificationConstantHelper.success.published);
   const contentData = await umbracoApi.document.getByName(contentName);
   expect(contentData.values[0].alias).toEqual(AliasHelper.toAlias(customDataTypeName));

tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Content/ContentWithCustomDataType.spec.ts

@@ -83,7 +83,8 @@ test('can create content with the custom data type with decimal property editor'
   await umbracoUi.content.isErrorNotificationVisible(false);
   expect(await umbracoApi.document.doesNameExist(contentName)).toBeTruthy();
   const contentData = await umbracoApi.document.getByName(contentName);
-  expect(contentData.values).toEqual([]);
+  expect(contentData.values[0].alias).toEqual(AliasHelper.toAlias(customDataTypeName));
+  expect(contentData.values[0].value).toEqual(0);
 });
 
 test('can add decimal number to the decimal in the content section', async ({umbracoApi, umbracoUi}) => {

tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Content/ContentWithDocumentTypeProperties/ContentWithAllowVaryByCulture.spec.ts

@@ -30,7 +30,7 @@ test('can create content with allow vary by culture enabled', async ({umbracoApi
   await umbracoUi.content.chooseDocumentType(documentTypeName);
   await umbracoUi.content.enterContentName(contentName);
   await umbracoUi.content.clickSaveButtonForContent();
-  await umbracoUi.content.clickSaveAndCloseButton();
+  await umbracoUi.content.clickSaveButton();
 
   // Assert
   //await umbracoUi.content.isSuccessNotificationVisible();
@@ -48,14 +48,13 @@ test('can create content with names that vary by culture', async ({umbracoApi, u
 
   // Act
   await umbracoUi.content.goToContentWithName(contentName);
-  await umbracoUi.content.clickVariantSelectorButton();
-  await umbracoUi.content.clickVariantAddModeButton();
+  await umbracoUi.content.clickSelectVariantButton();
+  await umbracoUi.content.clickVariantAddModeButtonForLanguageName(secondLanguageName);
   await umbracoUi.content.enterContentName(danishContentName);
   await umbracoUi.content.clickSaveButtonForContent();
-  await umbracoUi.content.clickSaveAndCloseButton();
+  await umbracoUi.content.clickSaveButton();
 
   // Assert
-  //await umbracoUi.content.isSuccessNotificationVisible();
   await umbracoUi.content.isErrorNotificationVisible(false);
   expect(await umbracoApi.document.doesNameExist(danishContentName)).toBeTruthy();
   const contentData = await umbracoApi.document.getByName(danishContentName);
@@ -68,7 +67,6 @@ test('can create content with names that vary by culture and content that is inv
   // Arrange
   const danishContentName = 'Test indhold';
   const textContent = 'This is a test text';
-  const danishTextContent = 'Dette er testtekst';
   const dataTypeName = 'Textstring';
   const dataTypeData = await umbracoApi.dataType.getByName(dataTypeName);
   const documentTypeId = await umbracoApi.documentType.createDocumentTypeWithPropertyEditor(documentTypeName, dataTypeName, dataTypeData.id, 'Test Group', true, false);
@@ -78,23 +76,21 @@ test('can create content with names that vary by culture and content that is inv
 
   // Act
   await umbracoUi.content.goToContentWithName(contentName);
-  await umbracoUi.content.clickVariantSelectorButton();
-  await umbracoUi.content.clickVariantAddModeButton();
+  await umbracoUi.content.clickSelectVariantButton();
+  await umbracoUi.content.clickVariantAddModeButtonForLanguageName(secondLanguageName);
   await umbracoUi.content.enterContentName(danishContentName);
-  await umbracoUi.content.enterTextstring(danishTextContent);
   await umbracoUi.content.clickSaveButtonForContent();
-  await umbracoUi.content.clickSaveAndCloseButton();
+  await umbracoUi.content.clickSaveButton();
 
   // Assert
-  //await umbracoUi.content.isSuccessNotificationVisible();
   await umbracoUi.content.isErrorNotificationVisible(false);
   expect(await umbracoApi.document.doesNameExist(danishContentName)).toBeTruthy();
   const contentData = await umbracoApi.document.getByName(danishContentName);
   expect(contentData.variants.length).toBe(2);
   expect(contentData.variants[0].name).toBe(contentName);
   expect(contentData.variants[1].name).toBe(danishContentName);
   expect(contentData.values.length).toBe(1);
-  expect(contentData.values[0].value).toBe(danishTextContent);
+  expect(contentData.values[0].value).toBe(textContent);
 });
 
 test('can create content with names and content that vary by culture', async ({umbracoApi, umbracoUi}) => {
@@ -111,12 +107,12 @@ test('can create content with names and content that vary by culture', async ({u
 
   // Act
   await umbracoUi.content.goToContentWithName(contentName);
-  await umbracoUi.content.clickVariantSelectorButton();
-  await umbracoUi.content.clickVariantAddModeButton();
+  await umbracoUi.content.clickSelectVariantButton();
+  await umbracoUi.content.clickVariantAddModeButtonForLanguageName(secondLanguageName);
   await umbracoUi.content.enterContentName(danishContentName);
   await umbracoUi.content.enterTextstring(danishTextContent);
   await umbracoUi.content.clickSaveButtonForContent();
-  await umbracoUi.content.clickSaveAndCloseButton();
+  await umbracoUi.content.clickSaveButton();
 
   // Assert
   //await umbracoUi.content.isSuccessNotificationVisible();

tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Content/ContentWithDropdown.spec.ts

@@ -103,17 +103,13 @@ test('can not publish a mandatory dropdown with an empty value', async ({umbraco
   // Do not select any dropdown values and the validation error appears
   await umbracoUi.content.clickSaveAndPublishButton();
   await umbracoUi.content.isValidationMessageVisible(ConstantHelper.validationMessages.emptyValue);
-  //await umbracoUi.content.doesSuccessNotificationHaveText(NotificationConstantHelper.success.saved);
-  await umbracoUi.content.isErrorNotificationVisible(false);
   await umbracoUi.content.doesErrorNotificationHaveText(NotificationConstantHelper.error.documentCouldNotBePublished);
   // Select a dropdown value and the validation error disappears
   await umbracoUi.content.chooseDropdownOption([optionValues[0]]);
   await umbracoUi.content.isValidationMessageVisible(ConstantHelper.validationMessages.emptyValue, false);
   await umbracoUi.content.clickSaveAndPublishButton();
 
   // Assert
-  //await umbracoUi.content.doesSuccessNotificationHaveText(NotificationConstantHelper.success.saved);
-  await umbracoUi.content.isErrorNotificationVisible(false);
   await umbracoUi.content.doesSuccessNotificationHaveText(NotificationConstantHelper.success.published);
   const contentData = await umbracoApi.document.getByName(contentName);
   expect(contentData.values[0].alias).toEqual(AliasHelper.toAlias(customDataTypeName));