Class TikTokenizerTokens

Namespace

FoundationaLLM.Common.Services.Tokenizers

Assembly

FoundationaLLM.Common.dll

Well-known special TikTokenizer token names.

public static class TikTokenizerTokens

Inheritance

object

TikTokenizerTokens

Inherited Members

object.Equals(object)

object.Equals(object, object)

object.GetHashCode()

object.GetType()

object.MemberwiseClone()

object.ReferenceEquals(object, object)

object.ToString()

Fields

ENDOFPROMPT

End of prompt token.

public const string ENDOFPROMPT = "<|endofprompt|>"

Field Value

string

ENDOFTEXT

End of text token.

public const string ENDOFTEXT = "<|endoftext|>"

Field Value

string

FIM_MIDDLE

Fill-In-the-Middle (FIM) middle token. For details see https://arxiv.org/abs/2207.14255 and https://github.com/openai/human-eval-infilling.

public const string FIM_MIDDLE = "<|fim_middle|>"

Field Value

string

FIM_PREFIX

Fill-In-the-Middle (FIM) prefix token. For details see https://arxiv.org/abs/2207.14255 and https://github.com/openai/human-eval-infilling.

public const string FIM_PREFIX = "<|fim_prefix|>"

Field Value

string

FIM_SUFFIX

Fill-In-the-Middle (FIM) suffix token. For details see https://arxiv.org/abs/2207.14255 and https://github.com/openai/human-eval-infilling.

public const string FIM_SUFFIX = "<|fim_suffix|>"

Field Value

string