Add support for BPE tokenization.

This commit is contained in:
2023-04-23 18:49:52 -07:00
parent 0fe35cd976
commit f61fe6a18f
9 changed files with 562 additions and 17 deletions

View File

@ -34,7 +34,8 @@ struct ServerCommand: ParsableCommand {
_ = Server.insecure(group: group)
.withServiceProviders([
ModelServiceProvider(modelManager: modelManager),
ImageGenerationServiceProvider(modelManager: modelManager)
ImageGenerationServiceProvider(modelManager: modelManager),
TokenizerServiceProvider(modelManager: modelManager)
])
.bind(host: bindHost, port: bindPort)