fix: synchronize resubmission of the same command buffer

EuphoricThinking · EuphoricThinking · commit 232146bb3f21 · 2025-05-30T03:14:39.000Z
synchronization implemented as the host thread waiting
for an event to be signalled after the completion of the command
buffer execution
diff --git a/unified-runtime/source/adapters/level_zero/v2/queue_immediate_in_order.cpp b/unified-runtime/source/adapters/level_zero/v2/queue_immediate_in_order.cpp
@@ -1025,6 +1025,11 @@ ur_result_t ur_queue_immediate_in_order_t::enqueueCommandBufferExp(
   ur_event_handle_t executionEvent =
       hCommandBuffer->getExecutionEventUnlocked();
 
+  if (executionEvent != nullptr) {
+    ZE2UR_CALL(zeEventHostSynchronize,
+               (executionEvent->getZeEvent(), UINT64_MAX));
+  }
+
   UR_CALL(enqueueGenericCommandListsExp(
       1, &commandBufferCommandList, phEvent, numEventsInWaitList,
       phEventWaitList, UR_COMMAND_ENQUEUE_COMMAND_BUFFER_EXP, executionEvent));
diff --git a/unified-runtime/test/conformance/exp_command_buffer/enqueue.cpp b/unified-runtime/test/conformance/exp_command_buffer/enqueue.cpp
@@ -17,23 +17,19 @@
 // enqueue_update.cpp test for a test verifying the order of submissions, as
 // the input/output to the kernels can be modified between the submissions.
 struct urEnqueueCommandBufferExpTest
-    : uur::command_buffer::urCommandBufferExpExecutionTestWithParam<ur_queue_flags_t> {
+    : uur::command_buffer::urCommandBufferExpExecutionTestWithParam<
+          ur_queue_flags_t> {
   virtual void SetUp() override {
     program_name = "increment";
-    UUR_RETURN_ON_FATAL_FAILURE(urCommandBufferExpExecutionTestWithParam::SetUp());
+    UUR_RETURN_ON_FATAL_FAILURE(
+        urCommandBufferExpExecutionTestWithParam::SetUp());
 
-    // Create an in-order queue
-    queue_type = std::get<1>(GetParam());
+    // Create an in-order or out-of-order queue, depending on the passed parameter
+    ur_queue_flags_t queue_type = std::get<1>(GetParam());
     ur_queue_properties_t queue_properties = {
-        UR_STRUCTURE_TYPE_QUEUE_PROPERTIES, nullptr, queue_type}; //GetParam()}; //0};
-    ASSERT_SUCCESS(
-        urQueueCreate(context, device, &queue_properties, &in_or_out_of_order_queue));
-
-    // // Create an out-of-order queue
-    // queue_properties.flags = UR_QUEUE_FLAG_OUT_OF_ORDER_EXEC_MODE_ENABLE;
-    // ASSERT_SUCCESS(
-    //     urQueueCreate(context, device, &queue_properties, &out_of_order_queue));
-    // ASSERT_NE(out_of_order_queue, nullptr);
+        UR_STRUCTURE_TYPE_QUEUE_PROPERTIES, nullptr, queue_type};
+    ASSERT_SUCCESS(urQueueCreate(context, device, &queue_properties,
+                                 &in_or_out_of_order_queue));
 
     ASSERT_SUCCESS(urUSMDeviceAlloc(context, device, nullptr, nullptr,
                                     allocation_size, &device_ptr));
@@ -44,16 +40,19 @@ struct urEnqueueCommandBufferExpTest
                                     &zero_pattern, allocation_size, 0, nullptr,
                                     nullptr));
 
-    std::vector<int> temp_val(buffer_size, 3);
-
     for (int i = 0; i < num_copy_buffers; i++) {
-      ASSERT_SUCCESS(urUSMDeviceAlloc(context, device, nullptr, nullptr, buffer_size * sizeof(int), (void**) &(dst_buffers[i])));
-      ASSERT_SUCCESS(urUSMDeviceAlloc(context, device, nullptr, nullptr, buffer_size * sizeof(int), (void**) &(src_buffers[i])));
-
-      ASSERT_SUCCESS(urEnqueueUSMMemcpy(in_or_out_of_order_queue, false, src_buffers[i], temp_val.data(), buffer_size * sizeof(int), 0, nullptr, nullptr));
+      ASSERT_SUCCESS(urUSMDeviceAlloc(context, device, nullptr, nullptr,
+                                      buffer_size * sizeof(int32_t),
+                                      (void **)&(dst_buffers[i])));
+      ASSERT_SUCCESS(urUSMDeviceAlloc(context, device, nullptr, nullptr,
+                                      buffer_size * sizeof(int32_t),
+                                      (void **)&(src_buffers[i])));
+
+      ASSERT_SUCCESS(urEnqueueUSMFill(
+          queue, src_buffers[i], sizeof(zero_pattern), &zero_pattern,
+          buffer_size * sizeof(int32_t), 0, nullptr, nullptr));
     }
 
-
     ASSERT_SUCCESS(urQueueFinish(queue));
 
     // Create command-buffer with a single kernel that does "Ptr[i] += 1;"
@@ -63,9 +62,12 @@ struct urEnqueueCommandBufferExpTest
         nullptr, 0, nullptr, 0, nullptr, 0, nullptr, nullptr, nullptr,
         nullptr));
 
-
+    // Schedule memory copying in order to prolong the buffer execution
     for (int i = 0; i < num_copy_buffers; i++) {
-       ASSERT_SUCCESS(urCommandBufferAppendUSMMemcpyExp(cmd_buf_handle, dst_buffers[i], src_buffers[i], buffer_size * sizeof(int), 0, nullptr, 0, nullptr, nullptr, nullptr, nullptr));
+      ASSERT_SUCCESS(urCommandBufferAppendUSMMemcpyExp(
+          cmd_buf_handle, dst_buffers[i], src_buffers[i],
+          buffer_size * sizeof(int32_t), 0, nullptr, 0, nullptr, nullptr,
+          nullptr, nullptr));
     }
 
     ASSERT_SUCCESS(urCommandBufferFinalizeExp(cmd_buf_handle));
@@ -76,10 +78,6 @@ struct urEnqueueCommandBufferExpTest
       EXPECT_SUCCESS(urQueueRelease(in_or_out_of_order_queue));
     }
 
-    // if (out_of_order_queue) {
-    //   EXPECT_SUCCESS(urQueueRelease(out_of_order_queue));
-    // }
-
     if (device_ptr) {
       EXPECT_SUCCESS(urUSMFree(context, device_ptr));
     }
@@ -94,13 +92,11 @@ struct urEnqueueCommandBufferExpTest
       }
     }
 
-    UUR_RETURN_ON_FATAL_FAILURE(urCommandBufferExpExecutionTestWithParam::TearDown());
+    UUR_RETURN_ON_FATAL_FAILURE(
+        urCommandBufferExpExecutionTestWithParam::TearDown());
   }
 
   ur_queue_handle_t in_or_out_of_order_queue = nullptr;
-  ur_queue_flags_t queue_type; // = 0;
-
-  // ur_queue_handle_t out_of_order_queue = nullptr;
 
   static constexpr size_t global_size = 16;
   static constexpr size_t global_offset = 0;
@@ -110,56 +106,52 @@ struct urEnqueueCommandBufferExpTest
 
   static constexpr int num_copy_buffers = 10;
   static constexpr int buffer_size = 512;
-  int* dst_buffers[num_copy_buffers];
-  int* src_buffers[num_copy_buffers];
+  int32_t *dst_buffers[num_copy_buffers] = {};
+  int32_t *src_buffers[num_copy_buffers] = {};
 };
 
-
 std::string deviceTestWithQueueTypePrinter(
-    const ::testing::TestParamInfo<std::tuple<uur::DeviceTuple, ur_queue_flags_t>> &info) {
+    const ::testing::TestParamInfo<
+        std::tuple<uur::DeviceTuple, ur_queue_flags_t>> &info) {
   auto device = std::get<0>(info.param).device;
   auto queue_type = std::get<1>(info.param);
 
   std::stringstream ss;
 
   switch (queue_type) {
-    case 0:
-      ss << "InOrderQueue";
-      break;
+  case 0:
+    ss << "InOrderQueue";
+    break;
 
-      // change to if and bitwise and?
-    case UR_QUEUE_FLAG_OUT_OF_ORDER_EXEC_MODE_ENABLE:
-      ss << "OutOfOrderQueue";
-      break;
+  case UR_QUEUE_FLAG_OUT_OF_ORDER_EXEC_MODE_ENABLE:
+    ss << "OutOfOrderQueue";
+    break;
 
-    default:
-      ss << "UnspecifiedQueueType" << queue_type;
+  default:
+    ss << "UnspecifiedQueueType" << queue_type;
   }
 
-
   return uur::GetPlatformAndDeviceName(device) + "__" +
          uur::GTestSanitizeString(ss.str());
 }
 
-// UUR_INSTANTIATE_DEVICE_TEST_SUITE(urEnqueueCommandBufferExpTest);
-UUR_DEVICE_TEST_SUITE_WITH_PARAM(urEnqueueCommandBufferExpTest, testing::Values(0, UR_QUEUE_FLAG_OUT_OF_ORDER_EXEC_MODE_ENABLE), deviceTestWithQueueTypePrinter);
+UUR_DEVICE_TEST_SUITE_WITH_PARAM(
+    urEnqueueCommandBufferExpTest,
+    testing::Values(0, UR_QUEUE_FLAG_OUT_OF_ORDER_EXEC_MODE_ENABLE),
+    deviceTestWithQueueTypePrinter);
 
 // Tests that the same command-buffer submitted across different in-order
 // queues has an implicit dependency on first submission
 TEST_P(urEnqueueCommandBufferExpTest, SerializeAcrossQueues) {
   // Execute command-buffer to first in-order queue (created by parent
-  // urQueueTest fixture)
-  if (queue_type == UR_QUEUE_FLAG_OUT_OF_ORDER_EXEC_MODE_ENABLE) {
-    GTEST_SKIP() << "unwantedoutoforder";
-  }
-
+  // urQueueTestWithParam fixture)
   ASSERT_SUCCESS(
       urEnqueueCommandBufferExp(queue, cmd_buf_handle, 0, nullptr, nullptr));
 
   // Execute command-buffer to second in-order queue, should have implicit
   // dependency on first submission.
-  ASSERT_SUCCESS(urEnqueueCommandBufferExp(in_or_out_of_order_queue, cmd_buf_handle, 0,
-                                           nullptr, nullptr));
+  ASSERT_SUCCESS(urEnqueueCommandBufferExp(
+      in_or_out_of_order_queue, cmd_buf_handle, 0, nullptr, nullptr));
 
   // Wait for both submissions to complete
   ASSERT_SUCCESS(urQueueFlush(queue));
@@ -177,47 +169,24 @@ TEST_P(urEnqueueCommandBufferExpTest, SerializeAcrossQueues) {
   }
 }
 
-// // Tests that submitting a command-buffer twice to an out-of-order queue
-// // relying on implicit serialization semantics for dependencies.
-// TEST_P(urEnqueueCommandBufferExpTest, SerializeOutofOrderQueue) {
-//   ASSERT_SUCCESS(urEnqueueCommandBufferExp(out_of_order_queue, cmd_buf_handle,
-//                                            0, nullptr, nullptr));
-//   ASSERT_SUCCESS(urEnqueueCommandBufferExp(out_of_order_queue, cmd_buf_handle,
-//                                            0, nullptr, nullptr));
-
-//   // Wait for both submissions to complete
-//   ASSERT_SUCCESS(urQueueFinish(out_of_order_queue));
-
-//   std::vector<uint32_t> Output(global_size);
-//   ASSERT_SUCCESS(urEnqueueUSMMemcpy(out_of_order_queue, true, Output.data(),
-//                                     device_ptr, allocation_size, 0, nullptr,
-//                                     nullptr));
-
-//   // Verify
-//   const uint32_t reference = 2;
-//   for (size_t i = 0; i < global_size; i++) {
-//     ASSERT_EQ(reference, Output[i]);
-//   }
-// }
-
 TEST_P(urEnqueueCommandBufferExpTest, SerializeInOrOutOfOrderQueue) {
   const int iterations = 5;
   for (int i = 0; i < iterations; i++) {
-    ASSERT_SUCCESS(urEnqueueCommandBufferExp(in_or_out_of_order_queue, cmd_buf_handle,
-                                           0, nullptr, nullptr));
+    ASSERT_SUCCESS(urEnqueueCommandBufferExp(
+        in_or_out_of_order_queue, cmd_buf_handle, 0, nullptr, nullptr));
   }
 
   // Wait for both submissions to complete
   ASSERT_SUCCESS(urQueueFinish(in_or_out_of_order_queue));
 
   std::vector<uint32_t> Output(global_size);
-  ASSERT_SUCCESS(urEnqueueUSMMemcpy(in_or_out_of_order_queue, true, Output.data(),
-                                    device_ptr, allocation_size, 0, nullptr,
-                                    nullptr));
+  ASSERT_SUCCESS(urEnqueueUSMMemcpy(in_or_out_of_order_queue, true,
+                                    Output.data(), device_ptr, allocation_size,
+                                    0, nullptr, nullptr));
 
   // Verify
   const uint32_t reference = iterations;
   for (size_t i = 0; i < global_size; i++) {
     ASSERT_EQ(reference, Output[i]);
   }
-}
+}
diff --git a/unified-runtime/test/conformance/exp_command_buffer/fixtures.h b/unified-runtime/test/conformance/exp_command_buffer/fixtures.h
@@ -98,7 +98,7 @@ struct urCommandBufferExpExecutionTest : uur::urKernelExecutionTest {
     UUR_RETURN_ON_FATAL_FAILURE(checkCommandBufferSupport(device));
 
     ur_exp_command_buffer_desc_t desc{UR_STRUCTURE_TYPE_EXP_COMMAND_BUFFER_DESC,
-                                      nullptr, false, false /* WAS: false */, false};
+                                      nullptr, false, false, false};
     ASSERT_SUCCESS(
         urCommandBufferCreateExp(context, device, &desc, &cmd_buf_handle));
     ASSERT_NE(cmd_buf_handle, nullptr);
@@ -115,9 +115,11 @@ struct urCommandBufferExpExecutionTest : uur::urKernelExecutionTest {
 };
 
 template <class T>
-struct urCommandBufferExpExecutionTestWithParam : urKernelExecutionTestWithParam<T> {
+struct urCommandBufferExpExecutionTestWithParam
+    : urKernelExecutionTestWithParam<T> {
   void SetUp() override {
-    UUR_RETURN_ON_FATAL_FAILURE(uur::urKernelExecutionTestWithParam<T>::SetUp());
+    UUR_RETURN_ON_FATAL_FAILURE(
+        uur::urKernelExecutionTestWithParam<T>::SetUp());
 
     UUR_RETURN_ON_FATAL_FAILURE(checkCommandBufferSupport(this->device));
 
@@ -132,7 +134,8 @@ struct urCommandBufferExpExecutionTestWithParam : urKernelExecutionTestWithParam
     if (cmd_buf_handle) {
       EXPECT_SUCCESS(urCommandBufferReleaseExp(cmd_buf_handle));
     }
-    UUR_RETURN_ON_FATAL_FAILURE(uur::urKernelExecutionTestWithParam<T>::TearDown());
+    UUR_RETURN_ON_FATAL_FAILURE(
+        uur::urKernelExecutionTestWithParam<T>::TearDown());
   }
 
   ur_exp_command_buffer_handle_t cmd_buf_handle = nullptr;
