replace self-exec with thread-based FSNotify worker

realrajaryan · realrajaryan · commit 4558603453bb · 2025-10-30T21:53:33.000-07:00
diff --git a/vminitd/Package.swift b/vminitd/Package.swift
@@ -47,7 +47,6 @@ let package = Package(
         .executableTarget(
             name: "vminitd",
             dependencies: [
-                .product(name: "ArgumentParser", package: "swift-argument-parser"),
                 .product(name: "Logging", package: "swift-log"),
                 .product(name: "_NIOFileSystem", package: "swift-nio"),
                 .product(name: "Containerization", package: "containerization"),
diff --git a/vminitd/Sources/vminitd/Application.swift b/vminitd/Sources/vminitd/Application.swift
@@ -14,7 +14,6 @@
 // limitations under the License.
 //===----------------------------------------------------------------------===//
 
-import ArgumentParser
 import Containerization
 import ContainerizationError
 import ContainerizationOS
@@ -26,46 +25,29 @@ import NIOPosix
 #if os(Linux)
 import Musl
 import LCShim
-#else
-import Darwin
 #endif
 
 @main
-struct Application: AsyncParsableCommand {
-    static let configuration = CommandConfiguration(
-        commandName: "vminitd",
-        abstract: "VM init process and container agent",
-        version: "1.0.0",
-        subcommands: [
-            InitCommand.self,
-            FsNotifyCommand.self,
-        ],
-        defaultSubcommand: InitCommand.self
-    )
+struct Application {
+    private static let foregroundEnvVar = "FOREGROUND"
+    private static let vsockPort = 1024
+    private static let standardErrorLock = NSLock()
 
-    static let foregroundEnvVar = "FOREGROUND"
-    static let vsockPort = 1024
-    static let standardErrorLock = NSLock()
-
-    static func runInForeground(_ log: Logger) throws {
-        precondition(getpid() != 1, "runInForeground must not be called as PID 1")
-        log.info("running vminitd under pid1 wrapper")
+    private static func runInForeground(_ log: Logger) throws {
+        log.info("running vminitd under pid1")
 
         var command = Command("/sbin/vminitd")
         command.attrs = .init(setsid: true)
         command.stdin = .standardInput
         command.stdout = .standardOutput
         command.stderr = .standardError
-
-        var env = ProcessInfo.processInfo.environment
-        env[foregroundEnvVar] = "1"
-        command.environment = env.map { "\($0.key)=\($0.value)" }
+        command.environment = ["\(foregroundEnvVar)=1"]
 
         try command.start()
         _ = try command.wait()
     }
 
-    static func adjustLimits() throws {
+    private static func adjustLimits() throws {
         var limits = rlimit()
         guard getrlimit(RLIMIT_NOFILE, &limits) == 0 else {
             throw POSIXError(.init(rawValue: errno)!)
@@ -78,232 +60,64 @@ struct Application: AsyncParsableCommand {
     }
 
     @Sendable
-    static func standardError(label: String) -> StreamLogHandler {
+    private static func standardError(label: String) -> StreamLogHandler {
         standardErrorLock.withLock {
             StreamLogHandler.standardError(label: label)
         }
     }
 
-    static func exit(_ code: Int32) -> Never {
-        #if os(Linux)
-        Musl.exit(code)
-        #else
-        Darwin.exit(code)
-        #endif
-    }
-}
-
-extension Application {
-    struct InitCommand: AsyncParsableCommand {
-        static let configuration = CommandConfiguration(
-            commandName: "init",
-            abstract: "Run vminitd as init process (default)"
-        )
-
-        func run() async throws {
-            LoggingSystem.bootstrap(Application.standardError)
-            var log = Logger(label: "vminitd")
-
-            try Application.adjustLimits()
-
-            // when running under debug mode, launch vminitd as a sub process of pid1
-            // so that we get a chance to collect better logs and errors before pid1 exists
-            // and the kernel panics.
-            #if DEBUG
-            let environment = ProcessInfo.processInfo.environment
-            let foreground = environment[Application.foregroundEnvVar]
-            let isPid1 = (getpid() == 1)
-            log.info("checking for shim var \(Application.foregroundEnvVar)=\(String(describing: foreground)); pid=\(getpid())")
+    static func main() async throws {
+        LoggingSystem.bootstrap(standardError)
+        var log = Logger(label: "vminitd")
 
-            // only use the FOREGROUND shim when we're not PID 1
-            // if we are PID 1 (fresh VM boot), skip the shim to avoid exiting init
-            if foreground == nil && !isPid1 {
-                try Application.runInForeground(log)
-                Application.exit(0)  // parent is not PID 1; safe to exit after child completes
-            }
+        try adjustLimits()
 
-            // we only need to be a subreaper when we're not PID 1
-            // (when PID 1, the kernel already reaps children)
-            if !isPid1 {
-                CZ_set_sub_reaper()
-            }
-            #endif
+        // when running under debug mode, launch vminitd as a sub process of pid1
+        // so that we get a chance to collect better logs and errors before pid1 exists
+        // and the kernel panics.
+        #if DEBUG
+        let environment = ProcessInfo.processInfo.environment
+        let foreground = environment[Self.foregroundEnvVar]
+        log.info("checking for shim var \(foregroundEnvVar)=\(String(describing: foreground))")
 
-            signal(SIGPIPE, SIG_IGN)
-
-            // Because the sysctl rpc wouldn't make sense if this didn't always exist, we
-            // ALWAYS mount /proc.
-            guard Musl.mount("proc", "/proc", "proc", 0, "") == 0 else {
-                log.error("failed to mount /proc")
-                Application.exit(1)
-            }
-            guard Musl.mount("tmpfs", "/run", "tmpfs", 0, "") == 0 else {
-                log.error("failed to mount /run")
-                Application.exit(1)
-            }
-            try Binfmt.mount()
-
-            log.logLevel = .debug
-
-            log.info("vminitd booting")
-            let eg = MultiThreadedEventLoopGroup(numberOfThreads: System.coreCount)
-            let server = Initd(log: log, group: eg)
-
-            do {
-                log.info("serving vminitd API")
-                try await server.serve(port: Application.vsockPort)
-                log.info("vminitd API returned")
-            } catch {
-                log.error("vminitd boot error \(error)")
-                Application.exit(1)
-            }
+        if foreground == nil {
+            try runInForeground(log)
+            exit(0)
         }
-    }
-}
-
-extension Application {
-    struct FsNotifyCommand: ParsableCommand {
-        static let configuration = CommandConfiguration(
-            commandName: "fs-notify",
-            abstract: "Internal command to run filesystem notification worker in container namespace",
-            shouldDisplay: false
-        )
-
-        @Argument(help: "Container PID whose namespace to enter")
-        var containerPID: Int32
-
-        private static let handshakeReady: UInt8 = 0xAA
-        private static let handshakeFailure: UInt8 = 0xFF
 
-        func run() throws {
-            // FD 3 = socket (extraFiles[0]), FD 4 = error pipe (extraFiles[1])
-            let socketFD: Int32 = 3
-            let errorPipeFD: Int32 = 4
-
-            do {
-                try enterContainerNamespace(containerPID: containerPID)
-                close(errorPipeFD)
-            } catch {
-                let errorMsg = "Failed to enter namespace: \(error)"
-                _ = errorMsg.utf8CString.withUnsafeBufferPointer { buffer in
-                    // -1 to skip null terminator
-                    write(errorPipeFD, buffer.baseAddress, buffer.count - 1)
-                }
-                close(errorPipeFD)
-
-                var failureHandshake = Self.handshakeFailure
-                _ = write(socketFD, &failureHandshake, 1)
-                close(socketFD)
-                Application.exit(1)
-            }
-
-            var readyHandshake = Self.handshakeReady
-            guard write(socketFD, &readyHandshake, 1) == 1 else {
-                close(socketFD)
-                Application.exit(1)
-            }
-
-            while true {
-                do {
-                    guard let (path, eventType) = try readEventFromParent(socket: socketFD) else {
-                        break
-                    }
-
-                    do {
-                        try generateSyntheticInotifyEvent(path: path, eventType: eventType)
-                    } catch {
-                        // Log detailed error to stderr (captured by parent)
-                        let errorMsg = "Failed to generate inotify event: path=\(path), type=\(eventType), error=\(error)"
-                        fputs(errorMsg + "\n", stderr)
-                        fflush(stderr)
-                    }
+        // since we are not running as pid1 in this mode we must set ourselves
+        // as a subpreaper so that all child processes are reaped by us and not
+        // passed onto our parent.
+        CZ_set_sub_reaper()
+        #endif
 
-                } catch {
-                    // Log and exit
-                    fputs("Protocol error reading from parent: \(error)\n", stderr)
-                    fflush(stderr)
-                    break
-                }
-            }
+        signal(SIGPIPE, SIG_IGN)
 
-            close(socketFD)
+        // Because the sysctl rpc wouldn't make sense if this didn't always exist, we
+        // ALWAYS mount /proc.
+        guard Musl.mount("proc", "/proc", "proc", 0, "") == 0 else {
+            log.error("failed to mount /proc")
+            exit(1)
         }
-
-        private func enterContainerNamespace(containerPID: Int32) throws {
-            let nsPath = "/proc/\(containerPID)/ns/mnt"
-            let vmNsPath = "/proc/self/ns/mnt"
-
-            let containerNsStatPtr = UnsafeMutablePointer<stat>.allocate(capacity: 1)
-            let vmNsStatPtr = UnsafeMutablePointer<stat>.allocate(capacity: 1)
-            defer {
-                containerNsStatPtr.deallocate()
-                vmNsStatPtr.deallocate()
-            }
-
-            let containerStatResult = stat(nsPath, containerNsStatPtr)
-            let vmStatResult = stat(vmNsPath, vmNsStatPtr)
-
-            if containerStatResult == 0 && vmStatResult == 0 {
-                let containerInode = containerNsStatPtr.pointee.st_ino
-                let vmInode = vmNsStatPtr.pointee.st_ino
-
-                if containerInode == vmInode {
-                    return
-                }
-            }
-
-            let fd = open(nsPath, O_RDONLY)
-            guard fd >= 0 else {
-                throw ContainerizationError(.internalError, message: "Failed to open namespace file: \(nsPath), errno \(errno)")
-            }
-            defer {
-                _ = close(fd)
-            }
-            let _ = unshare(CLONE_FS)
-            let setnsResult = setns(fd, CLONE_NEWNS)
-            guard setnsResult == 0 else {
-                throw ContainerizationError(.internalError, message: "Failed to setns to mount namespace: errno \(errno)")
-            }
+        guard Musl.mount("tmpfs", "/run", "tmpfs", 0, "") == 0 else {
+            log.error("failed to mount /run")
+            exit(1)
         }
+        try Binfmt.mount()
 
-        private func readEventFromParent(socket: Int32) throws -> (String, FileSystemEventType)? {
-            var eventTypeValue: UInt32 = 0
-            guard read(socket, &eventTypeValue, 4) == 4 else { return nil }
-            eventTypeValue = UInt32(bigEndian: eventTypeValue)
-
-            var pathLen: UInt32 = 0
-            guard read(socket, &pathLen, 4) == 4 else { return nil }
-            pathLen = UInt32(bigEndian: pathLen)
-
-            let pathData = UnsafeMutablePointer<UInt8>.allocate(capacity: Int(pathLen))
-            defer { pathData.deallocate() }
-            guard read(socket, pathData, Int(pathLen)) == pathLen else { return nil }
-            let pathBytes = Data(bytes: pathData, count: Int(pathLen))
-            guard let path = String(data: pathBytes, encoding: .utf8) else { return nil }
-
-            guard let eventType = FileSystemEventType(rawValue: Int(eventTypeValue)) else {
-                return nil
-            }
-
-            return (path, eventType)
-        }
+        log.logLevel = .debug
 
-        private func generateSyntheticInotifyEvent(
-            path: String,
-            eventType: FileSystemEventType
-        ) throws {
-            if eventType == .delete && !FileManager.default.fileExists(atPath: path) {
-                return
-            }
+        log.info("vminitd booting")
+        let eg = MultiThreadedEventLoopGroup(numberOfThreads: System.coreCount)
+        let server = Initd(log: log, group: eg)
 
-            let attributes = try FileManager.default.attributesOfItem(atPath: path)
-            guard let permissions = attributes[.posixPermissions] as? NSNumber else {
-                throw ContainerizationError(.internalError, message: "Failed to get file permissions for path: \(path)")
-            }
-            try FileManager.default.setAttributes(
-                [.posixPermissions: permissions],
-                ofItemAtPath: path
-            )
+        do {
+            log.info("serving vminitd API")
+            try await server.serve(port: vsockPort)
+            log.info("vminitd API returned")
+        } catch {
+            log.error("vminitd boot error \(error)")
+            exit(1)
         }
     }
 }
diff --git a/vminitd/Sources/vminitd/FilesystemEventWorker.swift b/vminitd/Sources/vminitd/FilesystemEventWorker.swift
