Add support for BPE tokenization.

Sources/StableDiffusionServer/TokenizerService.swift

@@ -0,0 +1,19 @@
+import Foundation
+import GRPC
+import StableDiffusionCore
+import StableDiffusionProtos
+
+class TokenizerServiceProvider: SdTokenizerServiceAsyncProvider {
+    private let modelManager: ModelManager
+
+    init(modelManager: ModelManager) {
+        self.modelManager = modelManager
+    }
+
+    func tokenize(request: SdTokenizeRequest, context _: GRPCAsyncServerCallContext) async throws -> SdTokenizeResponse {
+        guard let state = await modelManager.getModelState(name: request.modelName) else {
+            throw SdCoreError.modelNotFound
+        }
+        return try await state.tokenize(request)
+    }
+}