[SYCL] Simplify secondary queue usage

Following KhronosGroup/SYCL-Docs#811, the SYCL
2020 specification will not mandate the use nor exception checking of
secondary queues. This allows us to relax the interfaces taking a
secondary queue to fully ignore it. This commit drops the passing of
the secondary queue throughout the pipeline.

Signed-off-by: Larsen, Steffen <[email protected]>

Author: steffenlarsen

sycl/include/sycl/ext/oneapi/experimental/enqueue_functions.hpp

@@ -108,7 +108,7 @@ event submit_with_event_impl(queue &Q, PropertiesT Props,
                              CommandGroupFunc &&CGF,
                              const sycl::detail::code_location &CodeLoc) {
   return Q.submit_with_event<__SYCL_USE_FALLBACK_ASSERT>(
-      Props, detail::type_erased_cgfo_ty{CGF}, nullptr, CodeLoc);
+      Props, detail::type_erased_cgfo_ty{CGF}, CodeLoc);
 }
 } // namespace detail
 

sycl/include/sycl/handler.hpp

@@ -434,7 +434,6 @@ class __SYCL_EXPORT handler {
   handler(std::shared_ptr<detail::queue_impl> Queue, bool CallerNeedsEvent);
 #endif
 
-#ifdef __INTEL_PREVIEW_BREAKING_CHANGES
   /// Constructs SYCL handler from the pre-constructed handler_impl and the
   /// associated queue. Inside of Graph implementation, the Queue value is not
   /// used, for those cases it can be initialized with an empty shared_ptr.
@@ -443,7 +442,8 @@ class __SYCL_EXPORT handler {
   /// \param Queue is a SYCL queue.
   handler(detail::handler_impl *HandlerImpl,
           const std::shared_ptr<detail::queue_impl> &Queue);
-#else
+
+#ifndef __INTEL_PREVIEW_BREAKING_CHANGES
   /// Constructs SYCL handler from the associated queue and the submission's
   /// primary and secondary queue.
   ///
@@ -454,20 +454,16 @@ class __SYCL_EXPORT handler {
   ///        is null if no secondary queue is associated with the submission.
   /// \param CallerNeedsEvent indicates if the event resulting from this handler
   ///        is needed by the caller.
-#ifndef __INTEL_PREVIEW_BREAKING_CHANGES
   // TODO: This function is not used anymore, remove it in the next
   // ABI-breaking window.
   handler(std::shared_ptr<detail::queue_impl> Queue,
           std::shared_ptr<detail::queue_impl> PrimaryQueue,
           std::shared_ptr<detail::queue_impl> SecondaryQueue,
           bool CallerNeedsEvent);
-#endif
   __SYCL_DLL_LOCAL handler(std::shared_ptr<detail::queue_impl> Queue,
                            detail::queue_impl *SecondaryQueue,
                            bool CallerNeedsEvent);
-#endif
 
-#ifndef __INTEL_PREVIEW_BREAKING_CHANGES
   /// Constructs SYCL handler from Graph.
   ///
   /// The handler will add the command-group as a node to the graph rather than

sycl/include/sycl/queue.hpp

@@ -66,7 +66,7 @@ auto get_native(const SyclObjectT &Obj)
 namespace detail {
 class queue_impl;
 
-inline event submitAssertCapture(queue &, event &, queue *,
+inline event submitAssertCapture(queue &, event &,
                                  const detail::code_location &);
 
 // Function to postprocess submitted command
@@ -87,8 +87,10 @@ class __SYCL_EXPORT SubmissionInfo {
   sycl::detail::optional<SubmitPostProcessF> &PostProcessorFunc();
   const sycl::detail::optional<SubmitPostProcessF> &PostProcessorFunc() const;
 
+#ifndef __INTEL_PREVIEW_BREAKING_CHANGES
   std::shared_ptr<detail::queue_impl> &SecondaryQueue();
   const std::shared_ptr<detail::queue_impl> &SecondaryQueue() const;
+#endif
 
   ext::oneapi::experimental::event_mode_enum &EventMode();
   const ext::oneapi::experimental::event_mode_enum &EventMode() const;
@@ -438,17 +440,16 @@ class __SYCL_EXPORT queue : public detail::OwnerLessBase<queue> {
   /// for execution on a secondary queue.
   ///
   /// \param CGF is a function object containing command group.
-  /// \param SecondaryQueue is a fallback SYCL queue.
   /// \param CodeLoc is the code location of the submit call (default argument)
   /// \return a SYCL event object, which corresponds to the queue the command
   /// group is being enqueued on.
   template <typename T>
   std::enable_if_t<std::is_invocable_r_v<void, T, handler &>, event> submit(
-      T CGF, queue &SecondaryQueue,
+      T CGF, queue &,
       const detail::code_location &CodeLoc = detail::code_location::current()) {
     return submit_with_event<__SYCL_USE_FALLBACK_ASSERT>(
         sycl::ext::oneapi::experimental::empty_properties_t{},
-        detail::type_erased_cgfo_ty{CGF}, &SecondaryQueue, CodeLoc);
+        detail::type_erased_cgfo_ty{CGF}, CodeLoc);
   }
 
   /// Prevents any commands submitted afterward to this queue from executing
@@ -3582,7 +3583,7 @@ class __SYCL_EXPORT queue : public detail::OwnerLessBase<queue> {
       -> backend_return_t<BackendName, SyclObjectT>;
 
 #if __SYCL_USE_FALLBACK_ASSERT
-  friend event detail::submitAssertCapture(queue &, event &, queue *,
+  friend event detail::submitAssertCapture(queue &, event &,
                                            const detail::code_location &);
 #endif
 
@@ -3675,46 +3676,6 @@ class __SYCL_EXPORT queue : public detail::OwnerLessBase<queue> {
                                  const detail::code_location &CodeLoc,
                                  bool IsTopCodeLoc);
 
-  /// Submits a command group function object to the queue, in order to be
-  /// scheduled for execution on the device.
-  ///
-  /// \param Props is a property list with submission properties.
-  /// \param CGF is a function object containing command group.
-  /// \param SecondaryQueuePtr is a pointer to the secondary queue.
-  /// \param CodeLoc is the code location of the submit call (default argument)
-  /// \return a SYCL event object for the submitted command group.
-  //
-  // UseFallBackAssert as template param vs `#if` in function body is necessary
-  // to prevent ODR-violation between TUs built with different fallback assert
-  // modes.
-  template <bool UseFallbackAssert, typename PropertiesT>
-  event submit_with_event(
-      PropertiesT Props, const detail::type_erased_cgfo_ty &CGF,
-      queue *SecondaryQueuePtr,
-      const detail::code_location &CodeLoc = detail::code_location::current()) {
-    detail::tls_code_loc_t TlsCodeLocCapture(CodeLoc);
-    detail::v1::SubmissionInfo SI{};
-    ProcessSubmitProperties(Props, SI);
-    if (SecondaryQueuePtr)
-      SI.SecondaryQueue() = detail::getSyclObjImpl(*SecondaryQueuePtr);
-    if constexpr (UseFallbackAssert)
-      SI.PostProcessorFunc() =
-          [this, &SecondaryQueuePtr,
-           &TlsCodeLocCapture](bool IsKernel, bool KernelUsesAssert, event &E) {
-            if (IsKernel && !device_has(aspect::ext_oneapi_native_assert) &&
-                KernelUsesAssert && !device_has(aspect::accelerator)) {
-              // __devicelib_assert_fail isn't supported by Device-side Runtime
-              // Linking against fallback impl of __devicelib_assert_fail is
-              // performed by program manager class
-              // Fallback assert isn't supported for FPGA
-              submitAssertCapture(*this, E, SecondaryQueuePtr,
-                                  TlsCodeLocCapture.query());
-            }
-          };
-    return submit_with_event_impl(CGF, SI, TlsCodeLocCapture.query(),
-                                  TlsCodeLocCapture.isToplevel());
-  }
-
   /// Submits a command group function object to the queue, in order to be
   /// scheduled for execution on the device.
   ///
@@ -3743,7 +3704,7 @@ class __SYCL_EXPORT queue : public detail::OwnerLessBase<queue> {
           // Linking against fallback impl of __devicelib_assert_fail is
           // performed by program manager class
           // Fallback assert isn't supported for FPGA
-          submitAssertCapture(*this, E, nullptr, TlsCodeLocCapture.query());
+          submitAssertCapture(*this, E, TlsCodeLocCapture.query());
         }
       };
     return submit_with_event_impl(CGF, SI, TlsCodeLocCapture.query(),
@@ -3942,14 +3903,13 @@ class AssertInfoCopier;
  * Submit copy task for assert failure flag and host-task to check the flag
  * \param Event kernel's event to depend on i.e. the event represents the
  *              kernel to check for assertion failure
- * \param SecondaryQueue secondary queue for submit process, null if not used
  * \returns host tasks event
  *
  * This method doesn't belong to queue class to overcome msvc behaviour due to
  * which it gets compiled and exported without any integration header and, thus,
  * with no proper KernelInfo instance.
  */
-event submitAssertCapture(queue &Self, event &Event, queue *SecondaryQueue,
+event submitAssertCapture(queue &Self, event &Event,
                           const detail::code_location &CodeLoc) {
   buffer<detail::AssertHappened, 1> Buffer{1};
 
@@ -4005,10 +3965,10 @@ event submitAssertCapture(queue &Self, event &Event, queue *SecondaryQueue,
 
   CopierEv = Self.submit_with_event<true>(
       sycl::ext::oneapi::experimental::empty_properties_t{}, CopierCGF,
-      SecondaryQueue, CodeLoc);
+      CodeLoc);
   CheckerEv = Self.submit_with_event<true>(
       sycl::ext::oneapi::experimental::empty_properties_t{}, CheckerCGF,
-      SecondaryQueue, CodeLoc);
+      CodeLoc);
 
   return CheckerEv;
 }

sycl/source/detail/handler_impl.hpp

@@ -31,9 +31,7 @@ enum class HandlerSubmissionState : std::uint8_t {
 
 class handler_impl {
 public:
-  handler_impl(queue_impl *SubmissionSecondaryQueue, bool EventNeeded)
-      : MSubmissionSecondaryQueue(SubmissionSecondaryQueue),
-        MEventNeeded(EventNeeded) {};
+  handler_impl(bool EventNeeded) : MEventNeeded(EventNeeded) {};
 
   handler_impl(
       std::shared_ptr<ext::oneapi::experimental::detail::graph_impl> Graph)
@@ -67,10 +65,6 @@ class handler_impl {
   /// Registers mutually exclusive submission states.
   HandlerSubmissionState MSubmissionState = HandlerSubmissionState::NO_STATE;
 
-  /// Pointer to the secondary queue implementation. Nullptr if no
-  /// secondary queue fallback was given in the associated submission.
-  queue_impl *MSubmissionSecondaryQueue = nullptr;
-
   /// Bool stores information about whether the event resulting from the
   /// corresponding work is required.
   bool MEventNeeded = true;

sycl/source/detail/queue_impl.cpp

@@ -315,16 +315,16 @@ void queue_impl::addEvent(const event &Event) {
 
 event queue_impl::submit_impl(const detail::type_erased_cgfo_ty &CGF,
                               const std::shared_ptr<queue_impl> &Self,
-                              queue_impl *SecondaryQueue, bool CallerNeedsEvent,
+                              bool CallerNeedsEvent,
                               const detail::code_location &Loc,
                               bool IsTopCodeLoc,
                               const v1::SubmissionInfo &SubmitInfo) {
 #ifdef __INTEL_PREVIEW_BREAKING_CHANGES
-  detail::handler_impl HandlerImplVal(SecondaryQueue, CallerNeedsEvent);
+  detail::handler_impl HandlerImplVal(CallerNeedsEvent);
   detail::handler_impl *HandlerImpl = &HandlerImplVal;
   handler Handler(HandlerImpl, Self);
 #else
-  handler Handler(Self, SecondaryQueue, CallerNeedsEvent);
+  handler Handler(Self, CallerNeedsEvent);
   auto &HandlerImpl = detail::getSyclObjImpl(Handler);
 #endif
 
@@ -402,30 +402,15 @@ event queue_impl::submit_impl(const detail::type_erased_cgfo_ty &CGF,
       Stream->generateFlushCommand(ServiceCGH);
     };
     detail::type_erased_cgfo_ty CGF{L};
-    event FlushEvent =
-        submit_impl(CGF, Self, SecondaryQueue, /*CallerNeedsEvent*/ true, Loc,
-                    IsTopCodeLoc, {});
+    event FlushEvent = submit_impl(CGF, Self, /*CallerNeedsEvent*/ true, Loc,
+                                   IsTopCodeLoc, {});
     EventImpl->attachEventToCompleteWeak(detail::getSyclObjImpl(FlushEvent));
     registerStreamServiceEvent(detail::getSyclObjImpl(FlushEvent));
   }
 
   return Event;
 }
 
-#ifndef __INTEL_PREVIEW_BREAKING_CHANGES
-event queue_impl::submit_impl(const detail::type_erased_cgfo_ty &CGF,
-                              const std::shared_ptr<queue_impl> &Self,
-                              const std::shared_ptr<queue_impl> &,
-                              const std::shared_ptr<queue_impl> &SecondaryQueue,
-                              bool CallerNeedsEvent,
-                              const detail::code_location &Loc,
-                              bool IsTopCodeLoc,
-                              const SubmissionInfo &SubmitInfo) {
-  return submit_impl(CGF, Self, SecondaryQueue.get(), CallerNeedsEvent, Loc,
-                     IsTopCodeLoc, SubmitInfo);
-}
-#endif
-
 template <typename HandlerFuncT>
 event queue_impl::submitWithHandler(const std::shared_ptr<queue_impl> &Self,
                                     const std::vector<event> &DepEvents,

sycl/source/detail/queue_impl.hpp

@@ -319,19 +319,16 @@ class queue_impl {
   ///
   /// \param CGF is a function object containing command group.
   /// \param Self is a shared_ptr to this queue.
-  /// \param SecondQueue is a shared_ptr to the secondary queue.
   /// \param Loc is the code location of the submit call (default argument)
   /// \param StoreAdditionalInfo makes additional info be stored in event_impl
   /// \return a SYCL event object, which corresponds to the queue the command
   /// group is being enqueued on.
   event submit(const detail::type_erased_cgfo_ty &CGF,
                const std::shared_ptr<queue_impl> &Self,
-               const std::shared_ptr<queue_impl> &SecondQueue,
                const detail::code_location &Loc, bool IsTopCodeLoc,
                const SubmitPostProcessF *PostProcess = nullptr) {
     event ResEvent;
     v1::SubmissionInfo SI{};
-    SI.SecondaryQueue() = SecondQueue;
     if (PostProcess)
       SI.PostProcessorFunc() = *PostProcess;
     return submit_with_event(CGF, Self, SI, Loc, IsTopCodeLoc);
@@ -350,20 +347,17 @@ class queue_impl {
                           const std::shared_ptr<queue_impl> &Self,
                           const v1::SubmissionInfo &SubmitInfo,
                           const detail::code_location &Loc, bool IsTopCodeLoc) {
-
-    event ResEvent =
-        submit_impl(CGF, Self, SubmitInfo.SecondaryQueue().get(),
-                    /*CallerNeedsEvent=*/true, Loc, IsTopCodeLoc, SubmitInfo);
-    return ResEvent;
+    return submit_impl(CGF, Self, /*CallerNeedsEvent=*/true, Loc, IsTopCodeLoc,
+                       SubmitInfo);
   }
 
   void submit_without_event(const detail::type_erased_cgfo_ty &CGF,
                             const std::shared_ptr<queue_impl> &Self,
                             const v1::SubmissionInfo &SubmitInfo,
                             const detail::code_location &Loc,
                             bool IsTopCodeLoc) {
-    submit_impl(CGF, Self, SubmitInfo.SecondaryQueue().get(),
-                /*CallerNeedsEvent=*/false, Loc, IsTopCodeLoc, SubmitInfo);
+    submit_impl(CGF, Self, /*CallerNeedsEvent=*/false, Loc, IsTopCodeLoc,
+                SubmitInfo);
   }
 
   /// Performs a blocking wait for the completion of all enqueued tasks in the
@@ -879,43 +873,19 @@ class queue_impl {
     PostProcess(IsKernel, KernelUsesAssert, Event);
   }
 
-#ifndef __INTEL_PREVIEW_BREAKING_CHANGES
   /// Performs command group submission to the queue.
   ///
   /// \param CGF is a function object containing command group.
   /// \param Self is a pointer to this queue.
-  /// \param PrimaryQueue is a pointer to the primary queue. This may be the
-  ///        same as Self.
-  /// \param SecondaryQueue is a pointer to the secondary queue. This may be the
-  ///        same as Self.
   /// \param CallerNeedsEvent is a boolean indicating whether the event is
   ///        required by the user after the call.
   /// \param Loc is the code location of the submit call (default argument)
   /// \param SubmitInfo is additional optional information for the submission.
   /// \return a SYCL event representing submitted command group.
   event submit_impl(const detail::type_erased_cgfo_ty &CGF,
                     const std::shared_ptr<queue_impl> &Self,
-                    const std::shared_ptr<queue_impl> &PrimaryQueue,
-                    const std::shared_ptr<queue_impl> &SecondaryQueue,
                     bool CallerNeedsEvent, const detail::code_location &Loc,
-                    bool IsTopCodeLoc, const SubmissionInfo &SubmitInfo);
-#endif
-
-  /// Performs command group submission to the queue.
-  ///
-  /// \param CGF is a function object containing command group.
-  /// \param Self is a pointer to this queue.
-  /// \param SecondaryQueue is a pointer to the secondary queue.
-  /// \param CallerNeedsEvent is a boolean indicating whether the event is
-  ///        required by the user after the call.
-  /// \param Loc is the code location of the submit call (default argument)
-  /// \param SubmitInfo is additional optional information for the submission.
-  /// \return a SYCL event representing submitted command group.
-  event submit_impl(const detail::type_erased_cgfo_ty &CGF,
-                    const std::shared_ptr<queue_impl> &Self,
-                    queue_impl *SecondaryQueue, bool CallerNeedsEvent,
-                    const detail::code_location &Loc, bool IsTopCodeLoc,
-                    const v1::SubmissionInfo &SubmitInfo);
+                    bool IsTopCodeLoc, const v1::SubmissionInfo &SubmitInfo);
 
   /// Helper function for submitting a memory operation with a handler.
   /// \param Self is a shared_ptr to this queue.

sycl/source/detail/scheduler/scheduler.hpp

@@ -591,14 +591,6 @@ class Scheduler {
 
     void cleanupCommand(Command *Cmd, bool AllowUnsubmitted = false);
 
-    /// Reschedules the command passed using Queue provided.
-    ///
-    /// This can lead to rescheduling of all dependent commands. This can be
-    /// used when the user provides a "secondary" queue to the submit method
-    /// which may be used when the command fails to enqueue/execute in the
-    /// primary queue.
-    void rescheduleCommand(Command *Cmd, const QueueImplPtr &Queue);
-
     /// \return a pointer to the corresponding memory object record for the
     /// SYCL memory object provided, or nullptr if it does not exist.
     MemObjRecord *getMemObjRecord(SYCLMemObjI *MemObject);