network: configure mtu and segmentation offloading properly

container/src/init.rs

@@ -1,6 +1,7 @@
 use anyhow::{anyhow, Result};
 use futures::stream::TryStreamExt;
 use ipnetwork::IpNetwork;
+use krata::ethtool::EthtoolHandle;
 use krata::{LaunchInfo, LaunchNetwork};
 use log::{trace, warn};
 use nix::libc::{dup2, ioctl};
@@ -299,7 +300,12 @@ impl ContainerInit {
         let mut conf = lines.join("\n");
         conf.push('\n');
         fs::write(resolv, conf)?;
+        self.network_configure_ethtool(network).await?;
+        self.network_configure_link(network).await?;
+        Ok(())
+    }
 
+    async fn network_configure_link(&mut self, network: &LaunchNetwork) -> Result<()> {
         let (connection, handle, _) = rtnetlink::new_connection()?;
         tokio::spawn(connection);
 
@@ -365,7 +371,13 @@ impl ContainerInit {
                 warn!("failed to add ipv6 gateway route: {}", error);
             }
         }
+        Ok(())
+    }
 
+    async fn network_configure_ethtool(&mut self, network: &LaunchNetwork) -> Result<()> {
+        let mut handle = EthtoolHandle::new()?;
+        handle.set_gso(&network.link, false)?;
+        handle.set_tso(&network.link, false)?;
         Ok(())
     }
 

network/src/backend.rs

@@ -4,7 +4,7 @@ use crate::nat::Nat;
 use crate::proxynat::ProxyNatHandlerFactory;
 use crate::raw_socket::{AsyncRawSocketChannel, RawSocketHandle, RawSocketProtocol};
 use crate::vbridge::{BridgeJoinHandle, VirtualBridge};
-use crate::FORCE_MTU;
+use crate::EXTRA_MTU;
 use anyhow::{anyhow, Result};
 use bytes::BytesMut;
 use futures::TryStreamExt;
@@ -120,8 +120,9 @@ impl NetworkBackend {
             self.metadata.gateway.ipv4.into(),
             self.metadata.gateway.ipv6.into(),
         ];
-        let kdev = RawSocketHandle::bound_to_interface(&interface, RawSocketProtocol::Ethernet)?;
+        let mut kdev =
-        let mtu = FORCE_MTU;
+            RawSocketHandle::bound_to_interface(&interface, RawSocketProtocol::Ethernet)?;
+        let mtu = kdev.mtu_of_interface(&interface)? + EXTRA_MTU;
         let (tx_sender, tx_receiver) = channel::<BytesMut>(TX_CHANNEL_BUFFER_LEN);
         let mut udev = ChannelDevice::new(mtu, Medium::Ethernet, tx_sender.clone());
         let mac = self.metadata.gateway.mac;

network/src/lib.rs

@@ -22,7 +22,8 @@ pub mod proxynat;
 pub mod raw_socket;
 pub mod vbridge;
 
-pub const FORCE_MTU: usize = 65521;
+const HOST_BRIDGE_MTU: usize = 1500;
+pub const EXTRA_MTU: usize = 20;
 
 pub struct NetworkService {
     pub backends: HashMap<Uuid, JoinHandle<()>>,
@@ -33,7 +34,8 @@ pub struct NetworkService {
 impl NetworkService {
     pub async fn new() -> Result<NetworkService> {
         let bridge = VirtualBridge::new()?;
-        let hbridge = HostBridge::new(FORCE_MTU, "krata0".to_string(), &bridge).await?;
+        let hbridge =
+            HostBridge::new(HOST_BRIDGE_MTU + EXTRA_MTU, "krata0".to_string(), &bridge).await?;
         Ok(NetworkService {
             backends: HashMap::new(),
             bridge,

network/src/raw_socket.rs

@@ -255,7 +255,10 @@ impl AsyncRawSocketChannel {
                             }
                             let buffer = (&buffer[0..len]).into();
                             if let Err(error) = receive_sender.try_send(buffer) {
-                                debug!("raw socket failed to process received packet: {}", error);
+                                debug!(
+                                    "failed to process received packet from raw socket: {}",
+                                    error
+                                );
                             }
                         }
 
@@ -276,7 +279,11 @@ impl AsyncRawSocketChannel {
                                 debug!("failed to transmit: would block");
                                 continue;
                             }
-                            return Err(anyhow!("failed to write to raw socket: {}", error));
+                            return Err(anyhow!(
+                                "failed to write {} bytes to raw socket: {}",
+                                packet.len(),
+                                error
+                            ));
                         }
                     };
                 }

shared/Cargo.toml

@@ -5,7 +5,17 @@ edition = "2021"
 resolver = "2"
 
 [dependencies]
+anyhow = { workspace = true }
 serde = { workspace = true }
+libc = { workspace = true }
+
+[dependencies.nix]
+workspace = true
+features = ["ioctl", "socket"]
 
 [lib]
 path = "src/lib.rs"
+
+[[example]]
+name = "ethtool"
+path = "examples/ethtool.rs"

shared/examples/ethtool.rs

@@ -0,0 +1,13 @@
+use std::env;
+
+use anyhow::Result;
+use krata::ethtool::EthtoolHandle;
+
+fn main() -> Result<()> {
+    let args = env::args().collect::<Vec<String>>();
+    let interface = args.get(1).unwrap();
+    let mut handle = EthtoolHandle::new()?;
+    handle.set_gso(interface, false)?;
+    handle.set_tso(interface, false)?;
+    Ok(())
+}

shared/src/ethtool.rs

@@ -0,0 +1,77 @@
+use std::{
+    os::fd::{AsRawFd, FromRawFd, OwnedFd},
+    ptr::addr_of_mut,
+};
+
+use anyhow::Result;
+use libc::{ioctl, socket, AF_INET, SOCK_DGRAM};
+
+#[repr(C)]
+struct EthtoolValue {
+    cmd: u32,
+    data: u32,
+}
+
+const ETHTOOL_SGSO: u32 = 0x00000024;
+const ETHTOOL_STSO: u32 = 0x0000001f;
+const SIOCETHTOOL: libc::c_ulong = libc::SIOCETHTOOL;
+
+#[repr(C)]
+#[derive(Debug)]
+struct EthtoolIfreq {
+    ifr_name: [libc::c_char; libc::IF_NAMESIZE],
+    ifr_data: libc::uintptr_t,
+}
+
+impl EthtoolIfreq {
+    fn new(interface: &str) -> EthtoolIfreq {
+        let mut ifreq = EthtoolIfreq {
+            ifr_name: [0; libc::IF_NAMESIZE],
+            ifr_data: 0,
+        };
+        for (i, byte) in interface.as_bytes().iter().enumerate() {
+            ifreq.ifr_name[i] = *byte as libc::c_char
+        }
+        ifreq
+    }
+
+    fn set_value(&mut self, ptr: *mut libc::c_void) {
+        self.ifr_data = ptr as libc::uintptr_t;
+    }
+}
+
+pub struct EthtoolHandle {
+    fd: OwnedFd,
+}
+
+impl EthtoolHandle {
+    pub fn new() -> Result<EthtoolHandle> {
+        let fd = unsafe { socket(AF_INET, SOCK_DGRAM, 0) };
+        if fd == -1 {
+            return Err(std::io::Error::last_os_error().into());
+        }
+
+        Ok(EthtoolHandle {
+            fd: unsafe { OwnedFd::from_raw_fd(fd) },
+        })
+    }
+
+    pub fn set_gso(&mut self, interface: &str, value: bool) -> Result<()> {
+        self.set_value(interface, ETHTOOL_SGSO, if value { 1 } else { 0 })
+    }
+
+    pub fn set_tso(&mut self, interface: &str, value: bool) -> Result<()> {
+        self.set_value(interface, ETHTOOL_STSO, if value { 1 } else { 0 })
+    }
+
+    fn set_value(&mut self, interface: &str, cmd: u32, value: u32) -> Result<()> {
+        let mut ifreq = EthtoolIfreq::new(interface);
+        let mut value = EthtoolValue { cmd, data: value };
+        ifreq.set_value(addr_of_mut!(value) as *mut libc::c_void);
+        let result = unsafe { ioctl(self.fd.as_raw_fd(), SIOCETHTOOL, addr_of_mut!(ifreq) as u64) };
+        if result == -1 {
+            return Err(std::io::Error::last_os_error().into());
+        }
+        Ok(())
+    }
+}

shared/src/lib.rs

@@ -1,3 +1,5 @@
+pub mod ethtool;
+
 use serde::{Deserialize, Serialize};
 
 #[derive(Serialize, Deserialize, Debug)]