From 17889d1c643d0ec9d7bb22b3a7e22ab0f23937fb Mon Sep 17 00:00:00 2001 From: Alex Zenla Date: Mon, 4 Mar 2024 12:19:03 +0000 Subject: [PATCH] network: configure mtu and segmentation offloading properly --- container/src/init.rs | 12 ++++++ network/src/backend.rs | 7 ++-- network/src/lib.rs | 6 ++- network/src/raw_socket.rs | 11 +++++- shared/Cargo.toml | 10 +++++ shared/examples/ethtool.rs | 13 +++++++ shared/src/ethtool.rs | 77 ++++++++++++++++++++++++++++++++++++++ shared/src/lib.rs | 2 + 8 files changed, 131 insertions(+), 7 deletions(-) create mode 100644 shared/examples/ethtool.rs create mode 100644 shared/src/ethtool.rs diff --git a/container/src/init.rs b/container/src/init.rs index 31faefc..3d23945 100644 --- a/container/src/init.rs +++ b/container/src/init.rs @@ -1,6 +1,7 @@ use anyhow::{anyhow, Result}; use futures::stream::TryStreamExt; use ipnetwork::IpNetwork; +use krata::ethtool::EthtoolHandle; use krata::{LaunchInfo, LaunchNetwork}; use log::{trace, warn}; use nix::libc::{dup2, ioctl}; @@ -299,7 +300,12 @@ impl ContainerInit { let mut conf = lines.join("\n"); conf.push('\n'); fs::write(resolv, conf)?; + self.network_configure_ethtool(network).await?; + self.network_configure_link(network).await?; + Ok(()) + } + async fn network_configure_link(&mut self, network: &LaunchNetwork) -> Result<()> { let (connection, handle, _) = rtnetlink::new_connection()?; tokio::spawn(connection); @@ -365,7 +371,13 @@ impl ContainerInit { warn!("failed to add ipv6 gateway route: {}", error); } } + Ok(()) + } + async fn network_configure_ethtool(&mut self, network: &LaunchNetwork) -> Result<()> { + let mut handle = EthtoolHandle::new()?; + handle.set_gso(&network.link, false)?; + handle.set_tso(&network.link, false)?; Ok(()) } diff --git a/network/src/backend.rs b/network/src/backend.rs index 7d8ee46..43e784c 100644 --- a/network/src/backend.rs +++ b/network/src/backend.rs @@ -4,7 +4,7 @@ use crate::nat::Nat; use crate::proxynat::ProxyNatHandlerFactory; use crate::raw_socket::{AsyncRawSocketChannel, RawSocketHandle, RawSocketProtocol}; use crate::vbridge::{BridgeJoinHandle, VirtualBridge}; -use crate::FORCE_MTU; +use crate::EXTRA_MTU; use anyhow::{anyhow, Result}; use bytes::BytesMut; use futures::TryStreamExt; @@ -120,8 +120,9 @@ impl NetworkBackend { self.metadata.gateway.ipv4.into(), self.metadata.gateway.ipv6.into(), ]; - let kdev = RawSocketHandle::bound_to_interface(&interface, RawSocketProtocol::Ethernet)?; - let mtu = FORCE_MTU; + let mut kdev = + RawSocketHandle::bound_to_interface(&interface, RawSocketProtocol::Ethernet)?; + let mtu = kdev.mtu_of_interface(&interface)? + EXTRA_MTU; let (tx_sender, tx_receiver) = channel::(TX_CHANNEL_BUFFER_LEN); let mut udev = ChannelDevice::new(mtu, Medium::Ethernet, tx_sender.clone()); let mac = self.metadata.gateway.mac; diff --git a/network/src/lib.rs b/network/src/lib.rs index 68a82ac..0d2b1aa 100644 --- a/network/src/lib.rs +++ b/network/src/lib.rs @@ -22,7 +22,8 @@ pub mod proxynat; pub mod raw_socket; pub mod vbridge; -pub const FORCE_MTU: usize = 65521; +const HOST_BRIDGE_MTU: usize = 1500; +pub const EXTRA_MTU: usize = 20; pub struct NetworkService { pub backends: HashMap>, @@ -33,7 +34,8 @@ pub struct NetworkService { impl NetworkService { pub async fn new() -> Result { let bridge = VirtualBridge::new()?; - let hbridge = HostBridge::new(FORCE_MTU, "krata0".to_string(), &bridge).await?; + let hbridge = + HostBridge::new(HOST_BRIDGE_MTU + EXTRA_MTU, "krata0".to_string(), &bridge).await?; Ok(NetworkService { backends: HashMap::new(), bridge, diff --git a/network/src/raw_socket.rs b/network/src/raw_socket.rs index 58ba458..043caa6 100644 --- a/network/src/raw_socket.rs +++ b/network/src/raw_socket.rs @@ -255,7 +255,10 @@ impl AsyncRawSocketChannel { } let buffer = (&buffer[0..len]).into(); if let Err(error) = receive_sender.try_send(buffer) { - debug!("raw socket failed to process received packet: {}", error); + debug!( + "failed to process received packet from raw socket: {}", + error + ); } } @@ -276,7 +279,11 @@ impl AsyncRawSocketChannel { debug!("failed to transmit: would block"); continue; } - return Err(anyhow!("failed to write to raw socket: {}", error)); + return Err(anyhow!( + "failed to write {} bytes to raw socket: {}", + packet.len(), + error + )); } }; } diff --git a/shared/Cargo.toml b/shared/Cargo.toml index 0228ccb..0115c14 100644 --- a/shared/Cargo.toml +++ b/shared/Cargo.toml @@ -5,7 +5,17 @@ edition = "2021" resolver = "2" [dependencies] +anyhow = { workspace = true } serde = { workspace = true } +libc = { workspace = true } + +[dependencies.nix] +workspace = true +features = ["ioctl", "socket"] [lib] path = "src/lib.rs" + +[[example]] +name = "ethtool" +path = "examples/ethtool.rs" diff --git a/shared/examples/ethtool.rs b/shared/examples/ethtool.rs new file mode 100644 index 0000000..a577df7 --- /dev/null +++ b/shared/examples/ethtool.rs @@ -0,0 +1,13 @@ +use std::env; + +use anyhow::Result; +use krata::ethtool::EthtoolHandle; + +fn main() -> Result<()> { + let args = env::args().collect::>(); + let interface = args.get(1).unwrap(); + let mut handle = EthtoolHandle::new()?; + handle.set_gso(interface, false)?; + handle.set_tso(interface, false)?; + Ok(()) +} diff --git a/shared/src/ethtool.rs b/shared/src/ethtool.rs new file mode 100644 index 0000000..ce5b0b9 --- /dev/null +++ b/shared/src/ethtool.rs @@ -0,0 +1,77 @@ +use std::{ + os::fd::{AsRawFd, FromRawFd, OwnedFd}, + ptr::addr_of_mut, +}; + +use anyhow::Result; +use libc::{ioctl, socket, AF_INET, SOCK_DGRAM}; + +#[repr(C)] +struct EthtoolValue { + cmd: u32, + data: u32, +} + +const ETHTOOL_SGSO: u32 = 0x00000024; +const ETHTOOL_STSO: u32 = 0x0000001f; +const SIOCETHTOOL: libc::c_ulong = libc::SIOCETHTOOL; + +#[repr(C)] +#[derive(Debug)] +struct EthtoolIfreq { + ifr_name: [libc::c_char; libc::IF_NAMESIZE], + ifr_data: libc::uintptr_t, +} + +impl EthtoolIfreq { + fn new(interface: &str) -> EthtoolIfreq { + let mut ifreq = EthtoolIfreq { + ifr_name: [0; libc::IF_NAMESIZE], + ifr_data: 0, + }; + for (i, byte) in interface.as_bytes().iter().enumerate() { + ifreq.ifr_name[i] = *byte as libc::c_char + } + ifreq + } + + fn set_value(&mut self, ptr: *mut libc::c_void) { + self.ifr_data = ptr as libc::uintptr_t; + } +} + +pub struct EthtoolHandle { + fd: OwnedFd, +} + +impl EthtoolHandle { + pub fn new() -> Result { + let fd = unsafe { socket(AF_INET, SOCK_DGRAM, 0) }; + if fd == -1 { + return Err(std::io::Error::last_os_error().into()); + } + + Ok(EthtoolHandle { + fd: unsafe { OwnedFd::from_raw_fd(fd) }, + }) + } + + pub fn set_gso(&mut self, interface: &str, value: bool) -> Result<()> { + self.set_value(interface, ETHTOOL_SGSO, if value { 1 } else { 0 }) + } + + pub fn set_tso(&mut self, interface: &str, value: bool) -> Result<()> { + self.set_value(interface, ETHTOOL_STSO, if value { 1 } else { 0 }) + } + + fn set_value(&mut self, interface: &str, cmd: u32, value: u32) -> Result<()> { + let mut ifreq = EthtoolIfreq::new(interface); + let mut value = EthtoolValue { cmd, data: value }; + ifreq.set_value(addr_of_mut!(value) as *mut libc::c_void); + let result = unsafe { ioctl(self.fd.as_raw_fd(), SIOCETHTOOL, addr_of_mut!(ifreq) as u64) }; + if result == -1 { + return Err(std::io::Error::last_os_error().into()); + } + Ok(()) + } +} diff --git a/shared/src/lib.rs b/shared/src/lib.rs index 8dae750..7bfa964 100644 --- a/shared/src/lib.rs +++ b/shared/src/lib.rs @@ -1,3 +1,5 @@ +pub mod ethtool; + use serde::{Deserialize, Serialize}; #[derive(Serialize, Deserialize, Debug)]