Class LeasedTikTokenizer

Namespace

FoundationaLLM.Common.Services.Tokenizers

Assembly

FoundationaLLM.Common.dll

Extends the TikTokenizer class to support leasing to multiple clients.

public class LeasedTikTokenizer : TikTokenizer, ITokenizer

Inheritance

object

TikTokenizer

LeasedTikTokenizer

Implements

ITokenizer

Inherited Members

TikTokenizer.DefaultCacheSize

TikTokenizer.Encode(string, IReadOnlyCollection<string>)

TikTokenizer.Encode(string, bool)

TikTokenizer.EncodeTrimSuffix(string, IReadOnlyCollection<string>, int)

TikTokenizer.EncodeTrimSuffix(string, int, bool)

TikTokenizer.EncodeTrimPrefix(string, IReadOnlyCollection<string>, int)

TikTokenizer.EncodeTrimPrefix(string, int, bool)

TikTokenizer.Decode(int[])

TikTokenizer.NumOfCacheEntries

object.Equals(object)

object.Equals(object, object)

object.GetHashCode()

object.GetType()

object.MemberwiseClone()

object.ReferenceEquals(object, object)

object.ToString()

Extension Methods

ObjectExtensions.GetPropertyValues(object, List<string>)

Constructors

LeasedTikTokenizer(Stream, IReadOnlyDictionary<string, int>, string, int)

Extends the TikTokenizer class to support leasing to multiple clients.

public LeasedTikTokenizer(Stream tikTokenBpeFileStream, IReadOnlyDictionary<string, int> specialTokensEncoder, string pattern, int cacheSize)

Parameters

tikTokenBpeFileStream Stream

specialTokensEncoder IReadOnlyDictionary<string, int>

pattern string

cacheSize int

Properties

IsLeased

Indicates whether the tokenizer is leased or not.

public bool IsLeased { get; set; }

Property Value

bool