Do not execute multi-tile command list on single-tile command queue

Signed-off-by: Zbigniew Zdanowicz <[email protected]>

Author: zzdanowicz

level_zero/core/source/cmdqueue/cmdqueue.cpp

@@ -53,6 +53,7 @@ ze_result_t CommandQueueImp::initialize(bool copyOnly, bool isInternal) {
         commandStream->replaceGraphicsAllocation(bufferAllocation);
         isCopyOnlyCommandQueue = copyOnly;
         preemptionCmdSyncProgramming = getPreemptionCmdProgramming();
+        activeSubDevices = static_cast<uint32_t>(csr->getOsContext().getDeviceBitfield().count());
     }
     return returnValue;
 }
@@ -156,8 +157,9 @@ CommandQueue *CommandQueue::create(uint32_t productFamily, Device *device, NEO::
         }
     }
 
-    csr->getOsContext().ensureContextInitialized();
-    csr->initDirectSubmission(*device->getNEODevice(), csr->getOsContext());
+    auto &osContext = csr->getOsContext();
+    osContext.ensureContextInitialized();
+    csr->initDirectSubmission(*device->getNEODevice(), osContext);
     return commandQueue;
 }
 

level_zero/core/source/cmdqueue/cmdqueue.h

@@ -58,6 +58,7 @@ struct CommandQueue : _ze_command_queue_handle_t {
   protected:
     NEO::PreemptionMode commandQueuePreemptionMode = NEO::PreemptionMode::Initial;
     uint32_t partitionCount = 1;
+    uint32_t activeSubDevices = 1;
     bool preemptionCmdSyncProgramming = true;
     bool commandQueueDebugCmdsProgrammed = false;
     bool isCopyOnlyCommandQueue = false;

level_zero/core/source/cmdqueue/cmdqueue_hw.inl

@@ -88,6 +88,10 @@ ze_result_t CommandQueueHw<gfxCoreFamily>::executeCommandLists(
             return ZE_RESULT_ERROR_INVALID_COMMAND_LIST_TYPE;
         }
 
+        if (this->activeSubDevices < commandList->partitionCount) {
+            return ZE_RESULT_ERROR_INVALID_COMMAND_LIST_TYPE;
+        }
+
         if (commandList->containsCooperativeKernels()) {
             anyCommandListWithCooperativeKernels = true;
         } else {

level_zero/core/test/unit_tests/fixtures/device_fixture.cpp

@@ -138,5 +138,13 @@ void MultipleDevicesWithCustomHwInfo::SetUp() {
     driverHandle->initialize(std::move(devices));
 }
 
+void SingleRootMultiSubDeviceFixture::SetUp() {
+    MultiDeviceFixture::numRootDevices = 1u;
+    MultiDeviceFixture::SetUp();
+
+    device = driverHandle->devices[0];
+    neoDevice = device->getNEODevice();
+}
+
 } // namespace ult
 } // namespace L0

level_zero/core/test/unit_tests/fixtures/device_fixture.h

@@ -71,6 +71,13 @@ struct MultiDeviceFixture {
     L0::ContextImp *context = nullptr;
 };
 
+struct SingleRootMultiSubDeviceFixture : public MultiDeviceFixture {
+    void SetUp() override;
+
+    L0::Device *device = nullptr;
+    NEO::Device *neoDevice = nullptr;
+};
+
 struct ContextFixture : DeviceFixture {
     void SetUp() override;
     void TearDown() override;

level_zero/core/test/unit_tests/mocks/mock_cmdqueue.h

@@ -28,6 +28,7 @@ struct WhiteBox<::L0::CommandQueue> : public ::L0::CommandQueueImp {
     using BaseClass::submitBatchBuffer;
     using BaseClass::synchronizeByPollingForTaskCount;
     using BaseClass::taskCount;
+    using CommandQueue::activeSubDevices;
     using CommandQueue::commandQueuePreemptionMode;
     using CommandQueue::internalUsage;
     using CommandQueue::partitionCount;
@@ -86,6 +87,7 @@ struct MockCommandQueueHw : public L0::CommandQueueHw<gfxCoreFamily> {
     using BaseClass = ::L0::CommandQueueHw<gfxCoreFamily>;
     using BaseClass::commandStream;
     using BaseClass::printfFunctionContainer;
+    using L0::CommandQueue::activeSubDevices;
     using L0::CommandQueue::internalUsage;
     using L0::CommandQueue::partitionCount;
     using L0::CommandQueue::preemptionCmdSyncProgramming;

level_zero/core/test/unit_tests/sources/cmdqueue/test_cmdqueue.cpp

@@ -1592,6 +1592,7 @@ HWTEST_TEMPLATED_F(AubCsrTest, givenAubCsrWhenCallingExecuteCommandListsThenPoll
     L0::CommandQueue::fromHandle(commandQueue)->destroy();
 }
 using CommandQueueSynchronizeTest = Test<ContextFixture>;
+using MultiTileCommandQueueSynchronizeTest = Test<SingleRootMultiSubDeviceFixture>;
 
 template <typename GfxFamily>
 struct SynchronizeCsr : public NEO::UltCommandStreamReceiver<GfxFamily> {
@@ -1698,7 +1699,7 @@ HWTEST_F(CommandQueueSynchronizeTest, givenDebugOverrideEnabledWhenCallToSynchro
     L0::CommandQueue::fromHandle(commandQueue)->destroy();
 }
 
-HWTEST_F(CommandQueueSynchronizeTest, givenMultiplePartitionCountWhenCallingSynchronizeThenExpectTheSameNumberCsrSynchronizeCalls) {
+HWTEST2_F(MultiTileCommandQueueSynchronizeTest, givenMultiplePartitionCountWhenCallingSynchronizeThenExpectTheSameNumberCsrSynchronizeCalls, IsWithinXeGfxFamily) {
     const ze_command_queue_desc_t desc{};
     ze_result_t returnValue;
 
@@ -1717,13 +1718,15 @@ HWTEST_F(CommandQueueSynchronizeTest, givenMultiplePartitionCountWhenCallingSync
                                                            returnValue));
     EXPECT_EQ(returnValue, ZE_RESULT_SUCCESS);
     ASSERT_NE(nullptr, commandQueue);
+    EXPECT_EQ(2u, commandQueue->activeSubDevices);
 
     auto commandList = std::unique_ptr<CommandList>(whitebox_cast(CommandList::create(productFamily, device, NEO::EngineGroupType::RenderCompute, 0u, returnValue)));
     ASSERT_NE(nullptr, commandList);
     commandList->partitionCount = 2;
 
     ze_command_list_handle_t cmdListHandle = commandList->toHandle();
-    commandQueue->executeCommandLists(1, &cmdListHandle, nullptr, false);
+    returnValue = commandQueue->executeCommandLists(1, &cmdListHandle, nullptr, false);
+    EXPECT_EQ(returnValue, ZE_RESULT_SUCCESS);
 
     uint64_t timeout = std::numeric_limits<uint64_t>::max();
     commandQueue->synchronize(timeout);
@@ -1733,7 +1736,7 @@ HWTEST_F(CommandQueueSynchronizeTest, givenMultiplePartitionCountWhenCallingSync
     L0::CommandQueue::fromHandle(commandQueue)->destroy();
 }
 
-HWTEST_F(CommandQueueSynchronizeTest, givenCsrHasMultipleActivePartitionWhenExecutingCmdListOnNewCmdQueueThenExpectCmdPartitionCountMatchCsrActivePartitions) {
+HWTEST2_F(MultiTileCommandQueueSynchronizeTest, givenCsrHasMultipleActivePartitionWhenExecutingCmdListOnNewCmdQueueThenExpectCmdPartitionCountMatchCsrActivePartitions, IsWithinXeGfxFamily) {
     const ze_command_queue_desc_t desc{};
     ze_result_t returnValue;
 
@@ -1753,18 +1756,46 @@ HWTEST_F(CommandQueueSynchronizeTest, givenCsrHasMultipleActivePartitionWhenExec
                                                            returnValue));
     EXPECT_EQ(returnValue, ZE_RESULT_SUCCESS);
     ASSERT_NE(nullptr, commandQueue);
+    EXPECT_EQ(2u, commandQueue->activeSubDevices);
 
     auto commandList = std::unique_ptr<CommandList>(whitebox_cast(CommandList::create(productFamily, device, NEO::EngineGroupType::RenderCompute, 0u, returnValue)));
     ASSERT_NE(nullptr, commandList);
 
     ze_command_list_handle_t cmdListHandle = commandList->toHandle();
     commandQueue->executeCommandLists(1, &cmdListHandle, nullptr, false);
+    EXPECT_EQ(returnValue, ZE_RESULT_SUCCESS);
 
     EXPECT_EQ(2u, commandQueue->partitionCount);
 
     L0::CommandQueue::fromHandle(commandQueue)->destroy();
 }
 
+HWTEST_F(CommandQueueSynchronizeTest, givenSingleTileCsrWhenExecutingMultiTileCommandListThenExpectErrorOnExecute) {
+    const ze_command_queue_desc_t desc{};
+    ze_result_t returnValue;
+
+    auto commandQueue = whitebox_cast(CommandQueue::create(productFamily,
+                                                           device,
+                                                           neoDevice->getDefaultEngine().commandStreamReceiver,
+                                                           &desc,
+                                                           false,
+                                                           false,
+                                                           returnValue));
+    EXPECT_EQ(returnValue, ZE_RESULT_SUCCESS);
+    ASSERT_NE(nullptr, commandQueue);
+    EXPECT_EQ(1u, commandQueue->activeSubDevices);
+
+    auto commandList = std::unique_ptr<CommandList>(whitebox_cast(CommandList::create(productFamily, device, NEO::EngineGroupType::RenderCompute, 0u, returnValue)));
+    ASSERT_NE(nullptr, commandList);
+    commandList->partitionCount = 2;
+
+    ze_command_list_handle_t cmdListHandle = commandList->toHandle();
+    returnValue = commandQueue->executeCommandLists(1, &cmdListHandle, nullptr, false);
+    EXPECT_EQ(returnValue, ZE_RESULT_ERROR_INVALID_COMMAND_LIST_TYPE);
+
+    L0::CommandQueue::fromHandle(commandQueue)->destroy();
+}
+
 template <typename GfxFamily>
 struct TestCmdQueueCsr : public NEO::UltCommandStreamReceiver<GfxFamily> {
     TestCmdQueueCsr(const NEO::ExecutionEnvironment &executionEnvironment, const DeviceBitfield deviceBitfield)

shared/test/common/test_macros/header/test.h

@@ -1116,6 +1116,8 @@ using IsAtLeastGen12lp = IsAtLeastGfxCore<IGFX_GEN12LP_CORE>;
 using IsAtLeastXeHpCore = IsAtLeastGfxCore<IGFX_XE_HP_CORE>;
 using IsAtMostXeHpCore = IsAtMostGfxCore<IGFX_XE_HP_CORE>;
 
+using IsWithinXeGfxFamily = IsWithinGfxCore<IGFX_XE_HP_CORE, IGFX_XE_HP_CORE>;
+
 using IsADLS = IsProduct<IGFX_ALDERLAKE_S>;
 using IsBXT = IsProduct<IGFX_BROXTON>;
 using IsCFL = IsProduct<IGFX_COFFEELAKE>;