Merge branch 'feature/unicore_bootloader_can_run_multicore_app' into 'master'

esp_system: Fix case when multicore app can not be run if bootloader is unicore

Closes IDFGH-9336

See merge request espressif/esp-idf!22664

Author: KonstantinKondrashov

components/bootloader_support/src/bootloader_init.c

@@ -96,4 +96,12 @@ void bootloader_print_banner(void)
 #ifndef CONFIG_APP_REPRODUCIBLE_BUILD
     ESP_EARLY_LOGI(TAG, "compile time " __DATE__ " " __TIME__);
 #endif
+
+#if CONFIG_FREERTOS_UNICORE
+#if (SOC_CPU_CORES_NUM > 1)
+    ESP_EARLY_LOGW(TAG, "Unicore bootloader");
+#endif
+#else
+    ESP_EARLY_LOGI(TAG, "Multicore bootloader");
+#endif
 }

components/esp_system/port/cpu_start.c

@@ -42,6 +42,7 @@
 #include "soc/assist_debug_reg.h"
 #include "soc/system_reg.h"
 #include "esp32s3/rom/opi_flash.h"
+#include "hal/cache_hal.h"
 #elif CONFIG_IDF_TARGET_ESP32C3
 #include "esp32c3/rtc.h"
 #include "esp32c3/rom/cache.h"
@@ -84,6 +85,7 @@
 #include "esp_private/sleep_gpio.h"
 #include "hal/wdt_hal.h"
 #include "soc/rtc.h"
+#include "hal/cache_ll.h"
 #include "hal/efuse_ll.h"
 #include "soc/periph_defs.h"
 #include "esp_cpu.h"
@@ -256,6 +258,37 @@ static void start_other_core(void)
         esp_rom_delay_us(100);
     }
 }
+
+// This function is needed to make the multicore app runnable on a unicore bootloader (built with FREERTOS UNICORE).
+// It does some cache settings for other CPUs.
+void IRAM_ATTR do_multicore_settings(void)
+{
+    // We intentionally do not check the cache settings before changing them,
+    // because it helps to get the application to run on older bootloaders.
+#ifdef CONFIG_IDF_TARGET_ESP32
+    if (!efuse_ll_get_disable_app_cpu()) {
+        Cache_Read_Disable(1);
+        Cache_Flush(1);
+        DPORT_REG_SET_BIT(DPORT_APP_CACHE_CTRL1_REG, DPORT_APP_CACHE_MMU_IA_CLR);
+        DPORT_REG_CLR_BIT(DPORT_APP_CACHE_CTRL1_REG, DPORT_APP_CACHE_MMU_IA_CLR);
+        // We do not enable cache for CPU1 now because it will be done later in start_other_core().
+    }
+#endif
+
+    cache_bus_mask_t cache_bus_mask_core0 = cache_ll_l1_get_enabled_bus(0);
+#ifndef CONFIG_IDF_TARGET_ESP32
+    // 1. disable the cache before changing its settings.
+    cache_hal_disable(CACHE_TYPE_ALL);
+#endif
+    for (unsigned core = 1; core < SOC_CPU_CORES_NUM; core++) {
+        // 2. change cache settings. All cores must have the same settings.
+        cache_ll_l1_enable_bus(core, cache_bus_mask_core0);
+    }
+#ifndef CONFIG_IDF_TARGET_ESP32
+    // 3. enable the cache after changing its settings.
+    cache_hal_enable(CACHE_TYPE_ALL);
+#endif
+}
 #endif // !CONFIG_ESP_SYSTEM_SINGLE_CORE_MODE
 
 /*
@@ -312,6 +345,16 @@ void IRAM_ATTR call_start_cpu0(void)
     }
 #endif
 
+#if !CONFIG_APP_BUILD_TYPE_PURE_RAM_APP
+#if CONFIG_ESP_SYSTEM_SINGLE_CORE_MODE
+    ESP_EARLY_LOGI(TAG, "Unicore app");
+#else
+    ESP_EARLY_LOGI(TAG, "Multicore app");
+    // It helps to fix missed cache settings for other cores. It happens when bootloader is unicore.
+    do_multicore_settings();
+#endif
+#endif // !CONFIG_APP_BUILD_TYPE_PURE_RAM_APP
+
     // When the APP is loaded into ram for execution, some hardware initialization behaviors
     // in the bootloader are still necessary
 #if CONFIG_APP_BUILD_TYPE_RAM

components/hal/esp32/include/hal/cache_ll.h

@@ -99,6 +99,38 @@ static inline void cache_ll_l1_enable_bus(uint32_t cache_id, cache_bus_mask_t ma
     }
 }
 
+/**
+ * Returns enabled buses for a given core
+ *
+ * @param cache_id    cache ID (when l1 cache is per core)
+ *
+ * @return State of enabled buses
+ */
+__attribute__((always_inline))
+static inline cache_bus_mask_t cache_ll_l1_get_enabled_bus(uint32_t cache_id)
+{
+    cache_bus_mask_t mask = 0;
+    HAL_ASSERT(cache_id == 0 || cache_id == 1);
+    if (cache_id == 0) {
+        uint32_t bus_mask= DPORT_REG_READ(DPORT_PRO_CACHE_CTRL1_REG);
+        mask |= (!(bus_mask & DPORT_PRO_CACHE_MASK_IRAM0)) ? CACHE_BUS_IBUS0 : 0;
+        mask |= (!(bus_mask & DPORT_PRO_CACHE_MASK_IRAM1)) ? CACHE_BUS_IBUS1 : 0;
+        mask |= (!(bus_mask & DPORT_PRO_CACHE_MASK_IROM0)) ? CACHE_BUS_IBUS2 : 0;
+
+        mask |= (!(bus_mask & DPORT_PRO_CACHE_MASK_DROM0)) ? CACHE_BUS_DBUS0 : 0;
+        mask |= (!(bus_mask & DPORT_PRO_CACHE_MASK_DRAM1)) ? CACHE_BUS_DBUS1 : 0;
+    } else {
+        uint32_t bus_mask= DPORT_REG_READ(DPORT_APP_CACHE_CTRL1_REG);
+        mask |= (!(bus_mask & DPORT_APP_CACHE_MASK_IRAM0)) ? CACHE_BUS_IBUS0 : 0;
+        mask |= (!(bus_mask & DPORT_APP_CACHE_MASK_IRAM1)) ? CACHE_BUS_IBUS1 : 0;
+        mask |= (!(bus_mask & DPORT_APP_CACHE_MASK_IROM0)) ? CACHE_BUS_IBUS2 : 0;
+
+        mask |= (!(bus_mask & DPORT_APP_CACHE_MASK_DROM0)) ? CACHE_BUS_DBUS0 : 0;
+        mask |= (!(bus_mask & DPORT_APP_CACHE_MASK_DRAM1)) ? CACHE_BUS_DBUS1 : 0;
+    }
+    return mask;
+}
+
 /**
  * Disable the Cache Buses
  *

components/hal/esp32s3/include/hal/cache_ll.h

@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD
+ * SPDX-FileCopyrightText: 2022-2023 Espressif Systems (Shanghai) CO LTD
  *
  * SPDX-License-Identifier: Apache-2.0
  */
@@ -100,6 +100,37 @@ static inline void cache_ll_l1_enable_bus(uint32_t cache_id, cache_bus_mask_t ma
     REG_CLR_BIT(EXTMEM_DCACHE_CTRL1_REG, dbus_mask);
 }
 
+/**
+ * Returns enabled buses for a given core
+ *
+ * @param cache_id    cache ID (when l1 cache is per core)
+ *
+ * @return State of enabled buses
+ */
+__attribute__((always_inline))
+static inline cache_bus_mask_t cache_ll_l1_get_enabled_bus(uint32_t cache_id)
+{
+    cache_bus_mask_t mask = 0;
+    HAL_ASSERT(cache_id == 0 || cache_id == 1);
+    //On esp32s3, only `CACHE_BUS_IBUS0` and `CACHE_BUS_DBUS0` are supported. Use `cache_ll_l1_get_bus()` to get your bus first
+
+    uint32_t ibus_mask = REG_READ(EXTMEM_ICACHE_CTRL1_REG);
+    if (cache_id == 0) {
+        mask |= (!(ibus_mask & EXTMEM_ICACHE_SHUT_CORE0_BUS)) ? CACHE_BUS_IBUS0 : 0;
+    } else {
+        mask |= (!(ibus_mask & EXTMEM_ICACHE_SHUT_CORE1_BUS)) ? CACHE_BUS_IBUS0 : 0;
+    }
+
+    uint32_t dbus_mask = REG_READ(EXTMEM_DCACHE_CTRL1_REG);
+    if (cache_id == 1) {
+        mask |= (!(dbus_mask & EXTMEM_DCACHE_SHUT_CORE0_BUS)) ? CACHE_BUS_DBUS0 : 0;
+    } else {
+        mask |= (!(dbus_mask & EXTMEM_DCACHE_SHUT_CORE1_BUS)) ? CACHE_BUS_DBUS0 : 0;
+    }
+
+    return mask;
+}
+
 /**
  * Disable the Cache Buses
  *

tools/test_apps/.build-test-rules.yml

@@ -179,3 +179,8 @@ tools/test_apps/system/startup:
 tools/test_apps/system/test_watchpoint:
   enable:
     - if: IDF_TARGET in ["esp32", "esp32c3"] # Just one test per architecture
+
+tools/test_apps/system/unicore_bootloader:
+  enable:
+    - if: SOC_CPU_CORES_NUM > 1
+      reason: the test should be run on multicore chips

tools/test_apps/system/unicore_bootloader/CMakeLists.txt

@@ -0,0 +1,6 @@
+# The following lines of boilerplate have to be in your project's
+# CMakeLists in this exact order for cmake to work correctly
+cmake_minimum_required(VERSION 3.16)
+
+include($ENV{IDF_PATH}/tools/cmake/project.cmake)
+project(test_unicore_bootloader)

tools/test_apps/system/unicore_bootloader/README.md

@@ -0,0 +1,2 @@
+| Supported Targets | ESP32 | ESP32-S3 |
+| ----------------- | ----- | -------- |

tools/test_apps/system/unicore_bootloader/README.txt

@@ -0,0 +1,4 @@
+This project tests if the app can start up in a certain configuration.
+Multicore app can start up even if the bootloader is unicore.
+
+The test is only for Multicore chips.

tools/test_apps/system/unicore_bootloader/conftest.py

@@ -0,0 +1,52 @@
+# SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: Apache-2.0
+
+import logging
+import os
+
+import pytest
+from _pytest.fixtures import FixtureRequest
+from _pytest.monkeypatch import MonkeyPatch
+from pytest_embedded_idf.app import FlashFile
+from pytest_embedded_idf.serial import IdfSerial
+
+
+# This is a custom IdfSerial class to support custom functionality
+# which is required only for this test
+class FlashBootloader(IdfSerial):
+    def bootloader_flash(self, binary_path: str) -> None:
+        """
+        Flash bootloader.
+
+        :return: None
+        """
+        logging.info('Flashing bootloader')
+        bootloader_path = os.path.join(binary_path, 'bootloader', 'bootloader.bin')
+        logging.info(bootloader_path)
+        offs = int(self.app.sdkconfig.get('BOOTLOADER_OFFSET_IN_FLASH', 0))
+        logging.info('bootloader offset is {0}'.format(hex(offs)))
+        prev_flash_files = self.app.flash_files
+        flash_files = []
+        flash_files.append(
+            FlashFile(
+                offs,
+                bootloader_path,
+                False,
+            )
+        )
+        self.app.flash_files = flash_files
+        self.flash()
+        # Restore self.app.flash files to original value
+        self.app.flash_files = prev_flash_files
+
+
+@pytest.fixture(scope='module')
+def monkeypatch_module(request: FixtureRequest) -> MonkeyPatch:
+    mp = MonkeyPatch()
+    request.addfinalizer(mp.undo)
+    return mp
+
+
+@pytest.fixture(scope='module', autouse=True)
+def replace_dut_class(monkeypatch_module: MonkeyPatch) -> None:
+    monkeypatch_module.setattr('pytest_embedded_idf.IdfSerial', FlashBootloader)

tools/test_apps/system/unicore_bootloader/main/CMakeLists.txt

@@ -0,0 +1 @@
+idf_component_register(SRCS "main.c")

tools/test_apps/system/unicore_bootloader/main/main.c

@@ -0,0 +1,12 @@
+/*
+ * SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#include <stdio.h>
+
+void app_main(void)
+{
+    printf("App is running\n");
+}

tools/test_apps/system/unicore_bootloader/pytest_unicore_bootloader.py

@@ -0,0 +1,39 @@
+# SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: CC0-1.0
+
+import os
+
+import pytest
+from pytest_embedded import Dut
+
+
+@pytest.mark.esp32
+@pytest.mark.esp32s3
+@pytest.mark.generic
+@pytest.mark.parametrize('config', ['multicore'], indirect=True)
+def test_multicore_app_and_unicore_bootloader(dut: Dut) -> None:
+    dut.expect('Multicore bootloader')
+    dut.expect('Multicore app')
+    dut.expect('App is running')
+
+    path_to_unicore_build = os.path.join(dut.app.app_path, f'build_{dut.target}_unicore')
+    dut.serial.bootloader_flash(path_to_unicore_build)
+    dut.expect('Unicore bootloader')
+    dut.expect('Multicore app')
+    dut.expect('App is running')
+
+
+@pytest.mark.esp32
+@pytest.mark.esp32s3
+@pytest.mark.generic
+@pytest.mark.parametrize('config', ['unicore'], indirect=True)
+def test_unicore_app_and_multicore_bootloader(dut: Dut) -> None:
+    dut.expect('Unicore bootloader')
+    dut.expect('Unicore app')
+    dut.expect('App is running')
+
+    path_to_unicore_build = os.path.join(dut.app.app_path, f'build_{dut.target}_multicore')
+    dut.serial.bootloader_flash(path_to_unicore_build)
+    dut.expect('Multicore bootloader')
+    dut.expect('Unicore app')
+    dut.expect('App is running')

tools/test_apps/system/unicore_bootloader/sdkconfig.ci.multicore

@@ -0,0 +1 @@
+CONFIG_FREERTOS_UNICORE=y