Add support for BPE tokenization.

This commit is contained in:
2023-04-23 18:49:52 -07:00
parent 0fe35cd976
commit f61fe6a18f
9 changed files with 562 additions and 17 deletions

View File

@ -35,6 +35,8 @@ dependencies {
api("io.grpc:grpc-stub:1.54.1")
api("io.grpc:grpc-protobuf:1.54.1")
api("io.grpc:grpc-kotlin-stub:1.3.0")
implementation("com.google.protobuf:protobuf-java:3.22.3")
implementation("io.grpc:grpc-netty:1.54.1")
}