Add support for BPE tokenization.

Sources/StableDiffusionServer/TokenizerService.swift

@@ -0,0 +1,19 @@
+import Foundation
+import GRPC
+import StableDiffusionCore
+import StableDiffusionProtos
+
+class TokenizerServiceProvider: SdTokenizerServiceAsyncProvider {
+    private let modelManager: ModelManager
+
+    init(modelManager: ModelManager) {
+        self.modelManager = modelManager
+    }
+
+    func tokenize(request: SdTokenizeRequest, context _: GRPCAsyncServerCallContext) async throws -> SdTokenizeResponse {
+        guard let state = await modelManager.getModelState(name: request.modelName) else {
+            throw SdCoreError.modelNotFound
+        }
+        return try await state.tokenize(request)
+    }
+}

Sources/StableDiffusionServer/main.swift

@@ -34,7 +34,8 @@ struct ServerCommand: ParsableCommand {
         _ = Server.insecure(group: group)
             .withServiceProviders([
                 ModelServiceProvider(modelManager: modelManager),
-                ImageGenerationServiceProvider(modelManager: modelManager)
+                ImageGenerationServiceProvider(modelManager: modelManager),
+                TokenizerServiceProvider(modelManager: modelManager)
             ])
             .bind(host: bindHost, port: bindPort)