feat: Enable gdb debugging on x86

Enabling GDB support for debugging the guest kernel. This allows us to
connect a gdb server to firecracker and debug the guest.

Signed-off-by: Jack Thomson <[email protected]>

Author: JackThomson2

Cargo.lock

@@ -1238,6 +1238,30 @@
                     }
                 ]
             },
+            {
+                "syscall": "ioctl",
+                "args": [
+                    {
+                        "index": 1,
+                        "type": "dword",
+                        "op": "eq",
+                        "val": 1078505115,
+                        "comment": "KVM_SET_GUEST_DEBUG"
+                    }
+                ]
+            },
+            {
+                "syscall": "ioctl",
+                "args": [
+                    {
+                        "index": 1,
+                        "type": "dword",
+                        "op": "eq",
+                        "val": 3222843013,
+                        "comment": "KVM_TRANSLATE"
+                    }
+                ]
+            },
             {
                 "syscall": "ioctl",
                 "args": [

resources/seccomp/x86_64-unknown-linux-musl.json

@@ -49,6 +49,7 @@ serde_json = "1.0.128"
 
 [features]
 tracing = ["log-instrument", "seccompiler/tracing", "utils/tracing", "vmm/tracing"]
+debug = ["vmm/debug"]
 
 [lints]
 workspace = true

src/firecracker/Cargo.toml

@@ -19,6 +19,8 @@ crc64 = "2.0.0"
 derive_more = { version = "1.0.0", default-features = false, features = ["from", "display"] }
 displaydoc = "0.2.5"
 event-manager = "0.4.0"
+gdbstub = { version = "0.7.2", optional = true }
+gdbstub_arch = { version = "0.3.0", optional = true }
 kvm-bindings = { version = "0.9.1", features = ["fam-wrappers", "serde"] }
 kvm-ioctls = "0.18.0"
 lazy_static = "1.5.0"
@@ -56,7 +58,9 @@ itertools = "0.13.0"
 proptest = { version = "1.5.0", default-features = false, features = ["std"] }
 
 [features]
+default = []
 tracing = ["log-instrument"]
+debug = ["gdbstub", "gdbstub_arch"]
 
 [[bench]]
 name = "cpu_templates"

src/vmm/Cargo.toml

@@ -7,6 +7,8 @@
 use std::convert::TryFrom;
 use std::fmt::Debug;
 use std::io::{self, Seek, SeekFrom};
+#[cfg(feature = "debug")]
+use std::sync::mpsc;
 use std::sync::{Arc, Mutex};
 
 use event_manager::{MutEventSubscriber, SubscriberOps};
@@ -56,6 +58,8 @@ use crate::devices::virtio::net::Net;
 use crate::devices::virtio::rng::Entropy;
 use crate::devices::virtio::vsock::{Vsock, VsockUnixBackend};
 use crate::devices::BusDevice;
+#[cfg(feature = "debug")]
+use crate::gdb;
 use crate::logger::{debug, error};
 use crate::persist::{MicrovmState, MicrovmStateError};
 use crate::resources::VmResources;
@@ -128,6 +132,9 @@ pub enum StartMicrovmError {
     /// Error configuring ACPI: {0}
     #[cfg(target_arch = "x86_64")]
     Acpi(#[from] crate::acpi::AcpiError),
+    /// Error starting GDB debug session
+    #[cfg(feature = "debug")]
+    GdbServer(gdb::target::Error),
 }
 
 /// It's convenient to automatically convert `linux_loader::cmdline::Error`s
@@ -274,6 +281,13 @@ pub fn build_microvm_for_boot(
         cpu_template.kvm_capabilities.clone(),
     )?;
 
+    #[cfg(feature = "debug")]
+    let (gdb_tx, gdb_rx) = mpsc::channel();
+    #[cfg(feature = "debug")]
+    vcpus
+        .iter_mut()
+        .for_each(|vcpu| vcpu.attach_debug_info(gdb_tx.clone()));
+
     // The boot timer device needs to be the first device attached in order
     // to maintain the same MMIO address referenced in the documentation
     // and tests.
@@ -321,16 +335,28 @@ pub fn build_microvm_for_boot(
         boot_cmdline,
     )?;
 
+    let vmm = Arc::new(Mutex::new(vmm));
+
+    #[cfg(feature = "debug")]
+    if let Some(gdb_socket_addr) = &vm_resources.gdb_socket_addr {
+        gdb::server::gdb_thread(vmm.clone(), &vcpus, gdb_rx, entry_addr, gdb_socket_addr)
+            .map_err(GdbServer)?;
+    } else {
+        debug!("No GDB socket provided not starting gdb server.");
+    }
+
     // Move vcpus to their own threads and start their state machine in the 'Paused' state.
-    vmm.start_vcpus(
-        vcpus,
-        seccomp_filters
-            .get("vcpu")
-            .ok_or_else(|| MissingSeccompFilters("vcpu".to_string()))?
-            .clone(),
-    )
-    .map_err(VmmError::VcpuStart)
-    .map_err(Internal)?;
+    vmm.lock()
+        .unwrap()
+        .start_vcpus(
+            vcpus,
+            seccomp_filters
+                .get("vcpu")
+                .ok_or_else(|| MissingSeccompFilters("vcpu".to_string()))?
+                .clone(),
+        )
+        .map_err(VmmError::VcpuStart)
+        .map_err(Internal)?;
 
     // Load seccomp filters for the VMM thread.
     // Execution panics if filters cannot be loaded, use --no-seccomp if skipping filters
@@ -344,7 +370,6 @@ pub fn build_microvm_for_boot(
     .map_err(VmmError::SeccompFilters)
     .map_err(Internal)?;
 
-    let vmm = Arc::new(Mutex::new(vmm));
     event_manager.add_subscriber(vmm.clone());
 
     Ok(vmm)

src/vmm/src/builder.rs

@@ -0,0 +1,148 @@
+// Copyright 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+use std::os::unix::net::UnixStream;
+use std::sync::mpsc::Receiver;
+use std::sync::mpsc::TryRecvError::Empty;
+use std::sync::{Arc, Mutex};
+
+use gdbstub::common::{Signal, Tid};
+use gdbstub::conn::{Connection, ConnectionExt};
+use gdbstub::stub::run_blocking::{self, WaitForStopReasonError};
+use gdbstub::stub::{DisconnectReason, GdbStub, MultiThreadStopReason};
+use gdbstub::target::Target;
+use vm_memory::GuestAddress;
+
+use super::target::{vcpuid_to_tid, Error, FirecrackerTarget};
+use crate::logger::trace;
+use crate::Vmm;
+
+/// Starts the GDB event loop which acts as a proxy between the Vcpus and GDB
+pub fn event_loop(
+    connection: UnixStream,
+    vmm: Arc<Mutex<Vmm>>,
+    gdb_event_receiver: Receiver<usize>,
+    entry_addr: GuestAddress,
+) {
+    let target = FirecrackerTarget::new(vmm, gdb_event_receiver, entry_addr);
+    let connection: Box<dyn ConnectionExt<Error = std::io::Error>> = { Box::new(connection) };
+    let debugger = GdbStub::new(connection);
+
+    gdb_event_loop_thread(debugger, target);
+}
+
+struct GdbBlockingEventLoop {}
+
+impl run_blocking::BlockingEventLoop for GdbBlockingEventLoop {
+    type Target = FirecrackerTarget;
+    type Connection = Box<dyn ConnectionExt<Error = std::io::Error>>;
+
+    type StopReason = MultiThreadStopReason<u64>;
+
+    /// Poll for events from either Vcpu's or packets from the GDB connection
+    fn wait_for_stop_reason(
+        target: &mut FirecrackerTarget,
+        conn: &mut Self::Connection,
+    ) -> Result<
+        run_blocking::Event<MultiThreadStopReason<u64>>,
+        run_blocking::WaitForStopReasonError<
+            <Self::Target as Target>::Error,
+            <Self::Connection as Connection>::Error,
+        >,
+    > {
+        loop {
+            match target.gdb_event.try_recv() {
+                Ok(cpu_id) => {
+                    // The Vcpu reports it's id from raw_id so we straight convert here
+                    let tid = Tid::new(cpu_id).expect("Error converting cpu id to Tid");
+                    // If notify paused returns false this means we were already debugging a single
+                    // core, the target will track this for us to pick up later
+                    target.update_paused_vcpu(tid);
+                    trace!("Vcpu: {tid:?} paused from debug exit");
+
+                    let stop_reason = target
+                        .get_stop_reason(tid)
+                        .map_err(WaitForStopReasonError::Target)?;
+
+                    let Some(stop_response) = stop_reason else {
+                        // If we returned None this is a break which should be handled by
+                        // the guest kernel (e.g. kernel int3 self testing) so we won't notify
+                        // GDB and instead inject this back into the guest
+                        target
+                            .inject_bp_to_guest(tid)
+                            .map_err(WaitForStopReasonError::Target)?;
+                        target
+                            .request_resume(tid)
+                            .map_err(WaitForStopReasonError::Target)?;
+
+                        trace!("Injected BP into guest early exit");
+                        continue;
+                    };
+
+                    trace!("Returned stop reason to gdb: {stop_response:?}");
+                    return Ok(run_blocking::Event::TargetStopped(stop_response));
+                }
+                Err(Empty) => (),
+                Err(_) => {
+                    return Err(WaitForStopReasonError::Target(Error::GdbQueueError));
+                }
+            }
+
+            if conn.peek().map(|b| b.is_some()).unwrap_or(false) {
+                let byte = conn
+                    .read()
+                    .map_err(run_blocking::WaitForStopReasonError::Connection)?;
+                return Ok(run_blocking::Event::IncomingData(byte));
+            }
+        }
+    }
+
+    /// Invoked when the GDB client sends a Ctrl-C interrupt.
+    fn on_interrupt(
+        target: &mut FirecrackerTarget,
+    ) -> Result<Option<MultiThreadStopReason<u64>>, <FirecrackerTarget as Target>::Error> {
+        // notify the target that a ctrl-c interrupt has occurred.
+        let main_core = vcpuid_to_tid(0)?;
+
+        target.request_pause(main_core)?;
+        target.update_paused_vcpu(main_core);
+
+        let exit_reason = MultiThreadStopReason::SignalWithThread {
+            tid: main_core,
+            signal: Signal::SIGINT,
+        };
+        Ok(Some(exit_reason))
+    }
+}
+/// This loop will run while communication with GDB is in progress, after GDB disconnects we
+/// shutdown firecracker and the VM
+fn gdb_event_loop_thread(
+    debugger: GdbStub<FirecrackerTarget, Box<dyn ConnectionExt<Error = std::io::Error>>>,
+    mut target: FirecrackerTarget,
+) {
+    match debugger.run_blocking::<GdbBlockingEventLoop>(&mut target) {
+        Ok(disconnect_reason) => match disconnect_reason {
+            DisconnectReason::Disconnect => {
+                trace!("Client disconnected")
+            }
+            DisconnectReason::TargetExited(code) => {
+                trace!("Target exited with code {}", code)
+            }
+            DisconnectReason::TargetTerminated(sig) => {
+                trace!("Target terminated with signal {}", sig)
+            }
+            DisconnectReason::Kill => trace!("GDB sent a kill command"),
+        },
+        Err(e) => {
+            if e.is_target_error() {
+                trace!("target encountered a fatal error: {e:?}")
+            } else if e.is_connection_error() {
+                trace!("connection error: {e:?}")
+            } else {
+                trace!("gdbstub encountered a fatal error {e:?}")
+            }
+        }
+    }
+
+    target.shutdown();
+}