Merge branch 'v13/dev' into v13/contrib

Author: AndyButland

Directory.Packages.props

@@ -45,8 +45,8 @@
     <PackageVersion Include="Asp.Versioning.Mvc" Version="7.1.1" />
     <PackageVersion Include="Asp.Versioning.Mvc.ApiExplorer" Version="7.1.0" />
     <PackageVersion Include="Dazinator.Extensions.FileProviders" Version="2.0.0" />
-    <PackageVersion Include="Examine" Version="3.5.0" />
-    <PackageVersion Include="Examine.Core" Version="3.5.0" />
+    <PackageVersion Include="Examine" Version="3.7.0" />
+    <PackageVersion Include="Examine.Core" Version="3.7.0" />
     <PackageVersion Include="HtmlAgilityPack" Version="1.11.71" />
     <PackageVersion Include="K4os.Compression.LZ4" Version="1.3.8" />
     <PackageVersion Include="MailKit" Version="4.8.0" />

build/azure-pipelines.yml

@@ -71,7 +71,7 @@ stages:
       - job: A
         displayName: Build Umbraco CMS
         pool:
-          vmImage: 'ubuntu-latest'
+          vmImage: 'windows-latest'
         steps:
           - checkout: self
             submodules: false

src/Umbraco.Cms.Api.Delivery/Services/RequestPreviewService.cs

@@ -1,4 +1,4 @@
-using Microsoft.AspNetCore.Http;
+using Microsoft.AspNetCore.Http;
 using Umbraco.Cms.Core.DeliveryApi;
 
 namespace Umbraco.Cms.Api.Delivery.Services;
@@ -11,5 +11,5 @@ public RequestPreviewService(IHttpContextAccessor httpContextAccessor)
     }
 
     /// <inheritdoc />
-    public bool IsPreview() => GetHeaderValue("Preview") == "true";
+    public bool IsPreview() => string.Equals(GetHeaderValue("Preview"), "true", StringComparison.OrdinalIgnoreCase);
 }

src/Umbraco.Core/EmbeddedResources/Lang/da.xml

@@ -298,9 +298,6 @@
     <key alias="removeTextBox">Fjern denne tekstboks</key>
     <key alias="contentRoot">Indholdsrod</key>
     <key alias="includeUnpublished">Inkluder ikke-udgivet indhold.</key>
-    <key alias="forceRepublish">Udgiv uændrede elementer.</key>
-    <key alias="forceRepublishWarning">ADVARSEL: Udgivelse af alle sider under denne i indholdstræet, uanset om de er ændret eller ej, kan være en ressourcekrævende og langvarig proces.</key>
-    <key alias="forceRepublishAdvisory">Dette bør ikke være nødvendigt under normale omstændigheder, så fortsæt kun med denne handling, hvis du er sikker på, at det er nødvendigt.</key>
     <key alias="isSensitiveValue">Denne værdi er skjult. Hvis du har brug for adgang til at se denne værdi, bedes du
       kontakte din administrator.
     </key>

src/Umbraco.Core/EmbeddedResources/Lang/en.xml

@@ -309,9 +309,6 @@
     <key alias="removeTextBox">Remove this text box</key>
     <key alias="contentRoot">Content root</key>
     <key alias="includeUnpublished">Include unpublished content items.</key>
-    <key alias="forceRepublish">Publish unchanged items.</key>
-    <key alias="forceRepublishWarning">WARNING: Publishing all pages below this one in the content tree, whether or not they have changed, can be an expensive and long-running operation.</key>
-    <key alias="forceRepublishAdvisory">This should not be necessary in normal circumstances so please only proceed with this option selected if you are certain it is required.</key>
     <key alias="isSensitiveValue">This value is hidden. If you need access to view this value please contact your
       website administrator.
     </key>

src/Umbraco.Core/EmbeddedResources/Lang/en_us.xml

@@ -308,9 +308,6 @@
     <key alias="removeTextBox">Remove this text box</key>
     <key alias="contentRoot">Content root</key>
     <key alias="includeUnpublished">Include unpublished content items.</key>
-    <key alias="forceRepublish">Publish unchanged items.</key>
-    <key alias="forceRepublishWarning">WARNING: Publishing all pages below this one in the content tree, whether or not they have changed, can be an expensive and long-running operation.</key>
-    <key alias="forceRepublishAdvisory">This should not be necessary in normal circumstances so please only proceed with this option selected if you are certain it is required.</key>
     <key alias="isSensitiveValue">This value is hidden. If you need access to view this value please contact your
       website administrator.
     </key>

src/Umbraco.Core/Models/ContentEditing/ContentSaveAction.cs

@@ -59,43 +59,11 @@ public enum ContentSaveAction
     ///     Saves and publishes the content item including all descendants regardless of whether they have a published version
     ///     or not.
     /// </summary>
-    [Obsolete("This option is no longer used as the 'force' aspect has been extended into options for publishing unpublished and re-publishing changed content. Please use one of those options instead.")]
     PublishWithDescendantsForce = 10,
 
     /// <summary>
     ///     Creates and publishes the new content item including all descendants regardless of whether they have a published
     ///     version or not.
     /// </summary>
-    [Obsolete("This option is no longer used as the 'force' aspect has been extended into options for publishing unpublished and re-publishing changed content. Please use one of those options instead.")]
     PublishWithDescendantsForceNew = 11,
-
-    /// <summary>
-    ///     Saves and publishes the content item including all descendants including publishing previously unpublished content.
-    /// </summary>
-    PublishWithDescendantsIncludeUnpublished = 12,
-
-    /// <summary>
-    ///     Saves and publishes the new content item including all descendants including publishing previously unpublished content.
-    /// </summary>
-    PublishWithDescendantsIncludeUnpublishedNew = 13,
-
-    /// <summary>
-    ///     Saves and publishes the content item including all descendants irrespective of whether there are any pending changes.
-    /// </summary>
-    PublishWithDescendantsForceRepublish = 14,
-
-    /// <summary>
-    ///     Saves and publishes the new content item including all descendants including publishing previously unpublished content.
-    /// </summary>
-    PublishWithDescendantsForceRepublishNew = 15,
-
-    /// <summary>
-    ///     Saves and publishes the content item including all descendants including publishing previously unpublished content and irrespective of whether there are any pending changes.
-    /// </summary>
-    PublishWithDescendantsIncludeUnpublishedAndForceRepublish = 16,
-
-    /// <summary>
-    ///     Saves and publishes the new content item including all descendants including publishing previously unpublished content and irrespective of whether there are any pending changes.
-    /// </summary>
-    PublishWithDescendantsIncludeUnpublishedAndForceRepublishNew = 17,
 }

src/Umbraco.Core/PublishedCache/IPublishedSnapshotService.cs

@@ -31,6 +31,12 @@ public interface IPublishedSnapshotService : IDisposable
     /// </remarks>
     IPublishedSnapshot CreatePublishedSnapshot(string? previewToken);
 
+    /// <summary>
+    /// Indicates if the database cache is in the process of being rebuilt.
+    /// </summary>
+    /// <returns></returns>
+    bool IsRebuilding() => false;
+
     /// <summary>
     ///     Rebuilds internal database caches (but does not reload).
     /// </summary>
@@ -61,6 +67,38 @@ void Rebuild(
         IReadOnlyCollection<int>? mediaTypeIds = null,
         IReadOnlyCollection<int>? memberTypeIds = null);
 
+    /// <summary>
+    ///     Rebuilds internal database caches (but does not reload).
+    /// </summary>
+    /// <param name="contentTypeIds">
+    ///     If not null will process content for the matching content types, if empty will process all
+    ///     content
+    /// </param>
+    /// <param name="mediaTypeIds">
+    ///     If not null will process content for the matching media types, if empty will process all
+    ///     media
+    /// </param>
+    /// <param name="memberTypeIds">
+    ///     If not null will process content for the matching members types, if empty will process all
+    ///     members
+    /// </param>
+    /// <param name="useBackgroundThread">Flag indicating whether to use a background thread for the operation and immediately return to the caller.</param>
+    /// <remarks>
+    ///     <para>
+    ///         Forces the snapshot service to rebuild its internal database caches. For instance, some caches
+    ///         may rely on a database table to store pre-serialized version of documents.
+    ///     </para>
+    ///     <para>
+    ///         This does *not* reload the caches. Caches need to be reloaded, for instance via
+    ///         <see cref="DistributedCache" /> RefreshAllPublishedSnapshot method.
+    ///     </para>
+    /// </remarks>
+    void Rebuild(
+        bool useBackgroundThread,
+        IReadOnlyCollection<int>? contentTypeIds = null,
+        IReadOnlyCollection<int>? mediaTypeIds = null,
+        IReadOnlyCollection<int>? memberTypeIds = null) => Rebuild(contentTypeIds, mediaTypeIds, memberTypeIds);
+
 
     /// <summary>
     ///     Rebuilds all internal database caches (but does not reload).
@@ -77,6 +115,22 @@ void Rebuild(
     /// </remarks>
     void RebuildAll() => Rebuild(Array.Empty<int>(), Array.Empty<int>(), Array.Empty<int>());
 
+    /// <summary>
+    ///     Rebuilds all internal database caches (but does not reload).
+    /// </summary>
+    /// <param name="useBackgroundThread">Flag indicating whether to use a background thread for the operation and immediately return to the caller.</param>
+    /// <remarks>
+    ///     <para>
+    ///         Forces the snapshot service to rebuild its internal database caches. For instance, some caches
+    ///         may rely on a database table to store pre-serialized version of documents.
+    ///     </para>
+    ///     <para>
+    ///         This does *not* reload the caches. Caches need to be reloaded, for instance via
+    ///         <see cref="DistributedCache" /> RefreshAllPublishedSnapshot method.
+    ///     </para>
+    /// </remarks>
+    void RebuildAll(bool useBackgroundThread) => Rebuild(useBackgroundThread, Array.Empty<int>(), Array.Empty<int>(), Array.Empty<int>());
+
     /* An IPublishedCachesService implementation can rely on transaction-level events to update
      * its internal, database-level data, as these events are purely internal. However, it cannot
      * rely on cache refreshers CacheUpdated events to update itself, as these events are external

src/Umbraco.PublishedCache.NuCache/PublishedSnapshotService.cs

@@ -15,6 +15,7 @@
 using Umbraco.Cms.Core.Services;
 using Umbraco.Cms.Core.Services.Changes;
 using Umbraco.Cms.Core.Sync;
+using Umbraco.Cms.Infrastructure.HostedServices;
 using Umbraco.Cms.Infrastructure.PublishedCache.DataSource;
 using Umbraco.Cms.Infrastructure.PublishedCache.Persistence;
 using Umbraco.Extensions;
@@ -31,6 +32,9 @@ internal class PublishedSnapshotService : IPublishedSnapshotService
     // means faster execution, but uses memory - not sure if we want it
     // so making it configurable.
     public static readonly bool FullCacheWhenPreviewing = true;
+
+    private const string IsRebuildingDatabaseCacheRuntimeCacheKey = "temp_database_cache_rebuild_op";
+
     private readonly NuCacheSettings _config;
     private readonly ContentDataSerializer _contentDataSerializer;
     private readonly IDefaultCultureAccessor _defaultCultureAccessor;
@@ -51,6 +55,8 @@ internal class PublishedSnapshotService : IPublishedSnapshotService
     private readonly object _storesLock = new();
     private readonly ISyncBootStateAccessor _syncBootStateAccessor;
     private readonly IVariationContextAccessor _variationContextAccessor;
+    private readonly IBackgroundTaskQueue _backgroundTaskQueue;
+    private readonly IAppPolicyCache _runtimeCache;
 
     private long _contentGen;
 
@@ -91,7 +97,9 @@ public PublishedSnapshotService(
         IPublishedModelFactory publishedModelFactory,
         IHostingEnvironment hostingEnvironment,
         IOptions<NuCacheSettings> config,
-        ContentDataSerializer contentDataSerializer)
+        ContentDataSerializer contentDataSerializer,
+        IBackgroundTaskQueue backgroundTaskQueue,
+        AppCaches appCaches)
     {
         _options = options;
         _syncBootStateAccessor = syncBootStateAccessor;
@@ -111,6 +119,8 @@ public PublishedSnapshotService(
         _contentDataSerializer = contentDataSerializer;
         _config = config.Value;
         _publishedModelFactory = publishedModelFactory;
+        _backgroundTaskQueue = backgroundTaskQueue;
+        _runtimeCache = appCaches.RuntimeCache;
     }
 
     protected PublishedSnapshot? CurrentPublishedSnapshot
@@ -349,12 +359,66 @@ public IPublishedSnapshot CreatePublishedSnapshot(string? previewToken)
         return new PublishedSnapshot(this, preview);
     }
 
+    /// <inheritdoc />
+    public bool IsRebuilding() => _runtimeCache.Get(IsRebuildingDatabaseCacheRuntimeCacheKey) is not null;
+
     /// <inheritdoc />
     public void Rebuild(
         IReadOnlyCollection<int>? contentTypeIds = null,
         IReadOnlyCollection<int>? mediaTypeIds = null,
         IReadOnlyCollection<int>? memberTypeIds = null)
-        => _publishedContentService.Rebuild(contentTypeIds, mediaTypeIds, memberTypeIds);
+        => Rebuild(false, contentTypeIds, mediaTypeIds, memberTypeIds);
+
+    /// <inheritdoc />
+    public void Rebuild(
+        bool useBackgroundThread,
+        IReadOnlyCollection<int>? contentTypeIds = null,
+        IReadOnlyCollection<int>? mediaTypeIds = null,
+        IReadOnlyCollection<int>? memberTypeIds = null)
+    {
+        if (useBackgroundThread)
+        {
+            _logger.LogInformation("Starting async background thread for rebuilding database cache.");
+
+            _backgroundTaskQueue.QueueBackgroundWorkItem(
+                cancellationToken =>
+                {
+                    // Do not flow AsyncLocal to the child thread
+                    using (ExecutionContext.SuppressFlow())
+                    {
+                        Task.Run(() => PerformRebuild(contentTypeIds, mediaTypeIds, memberTypeIds));
+
+                        // immediately return so the request isn't waiting.
+                        return Task.CompletedTask;
+                    }
+                });
+        }
+        else
+        {
+            PerformRebuild(contentTypeIds, mediaTypeIds, memberTypeIds);
+        }
+    }
+
+    private void PerformRebuild(
+        IReadOnlyCollection<int>? contentTypeIds = null,
+        IReadOnlyCollection<int>? mediaTypeIds = null,
+        IReadOnlyCollection<int>? memberTypeIds = null)
+    {
+        try
+        {
+            SetIsRebuilding();
+
+            _publishedContentService.Rebuild(contentTypeIds, mediaTypeIds, memberTypeIds);
+        }
+        finally
+        {
+            ClearIsRebuilding();
+        }
+    }
+
+    private void SetIsRebuilding() => _runtimeCache.Insert(IsRebuildingDatabaseCacheRuntimeCacheKey, () => "tempValue", TimeSpan.FromMinutes(10));
+
+    private void ClearIsRebuilding() => _runtimeCache.Clear(IsRebuildingDatabaseCacheRuntimeCacheKey);
 
     public async Task CollectAsync()
     {

src/Umbraco.PublishedCache.NuCache/PublishedSnapshotStatus.cs

@@ -29,6 +29,11 @@ public string GetStatus()
                 $"The current {typeof(IPublishedSnapshotService)} is not the default type. A status cannot be determined.";
         }
 
+        if (_service.IsRebuilding())
+        {
+            return "Rebuild in progress. Please wait.";
+        }
+
         // TODO: This should be private
         _service.EnsureCaches();
 

src/Umbraco.Web.BackOffice/Authorization/ContentPermissionsQueryStringHandler.cs

@@ -20,6 +20,8 @@ public class
 {
     private readonly ContentPermissions _contentPermissions;
 
+    protected override UmbracoObjectTypes KeyParsingFilterType => UmbracoObjectTypes.Document;
+
     /// <summary>
     ///     Initializes a new instance of the <see cref="ContentPermissionsQueryStringHandler" /> class.
     /// </summary>
@@ -48,7 +50,11 @@ protected override Task<bool> IsAuthorized(AuthorizationHandlerContext context,
                 return Task.FromResult(true);
             }
 
-            var argument = routeVal.ToString();
+            // Handle case where the incoming querystring could contain more than one value (e.g. ?id=1000&id=1001).
+            // It's the first one that'll be processed by the protected method so we should verify that.
+            var argument = routeVal.Count == 1
+                ? routeVal.ToString()
+                : routeVal.FirstOrDefault()?.ToString() ?? string.Empty;
 
             if (!TryParseNodeId(argument, out nodeId))
             {

src/Umbraco.Web.BackOffice/Authorization/MediaPermissionsQueryStringHandler.cs

@@ -18,6 +18,8 @@ public class MediaPermissionsQueryStringHandler : PermissionsQueryStringHandler<
 {
     private readonly MediaPermissions _mediaPermissions;
 
+    protected override UmbracoObjectTypes KeyParsingFilterType => UmbracoObjectTypes.Media;
+
     /// <summary>
     ///     Initializes a new instance of the <see cref="MediaPermissionsQueryStringHandler" /> class.
     /// </summary>
@@ -44,7 +46,11 @@ protected override Task<bool> IsAuthorized(AuthorizationHandlerContext context,
             return Task.FromResult(true);
         }
 
-        var argument = routeVal.ToString();
+        // Handle case where the incoming querystring could contain more than one value (e.g. ?id=1000&id=1001).
+        // It's the first one that'll be processed by the protected method so we should verify that.
+        var argument = routeVal.Count == 1
+            ? routeVal.ToString()
+            : routeVal.FirstOrDefault()?.ToString() ?? string.Empty;
 
         if (!TryParseNodeId(argument, out var nodeId))
         {

src/Umbraco.Web.BackOffice/Authorization/PermissionsQueryStringHandler.cs

@@ -49,12 +49,18 @@ public PermissionsQueryStringHandler(
     /// </summary>
     protected IEntityService EntityService { get; set; }
 
+    /// <summary>
+    /// Defaults to Unknown so all types are allowed, since Keys are unique across all node types this works,
+    /// but it if you are certain you are looking for a specific type this should be overwritten for DB query performance.
+    /// </summary>
+    protected virtual UmbracoObjectTypes KeyParsingFilterType => UmbracoObjectTypes.Unknown;
+
     /// <summary>
     ///     Attempts to parse a node ID from a string representation found in a querystring value.
     /// </summary>
     /// <param name="argument">Querystring value.</param>
     /// <param name="nodeId">Output parsed Id.</param>
-    /// <returns>True of node ID could be parased, false it not.</returns>
+    /// <returns>True of node ID could be parsed, false it not.</returns>
     protected bool TryParseNodeId(string argument, out int nodeId)
     {
         // If the argument is an int, it will parse and can be assigned to nodeId.
@@ -75,7 +81,7 @@ protected bool TryParseNodeId(string argument, out int nodeId)
 
         if (Guid.TryParse(argument, out Guid key))
         {
-            nodeId = EntityService.GetId(key, UmbracoObjectTypes.Document).Result;
+            nodeId = EntityService.GetId(key, KeyParsingFilterType).Result;
             return true;
         }