interface TokenTextSplitterParams {
    allowedSpecial: string[] | "all";
    chunkOverlap: number;
    chunkSize: number;
    disallowedSpecial: string[] | "all";
    encodingName: TiktokenEncoding;
    keepSeparator: boolean;
    lengthFunction?: ((text: string) => number) | ((text: string) => Promise<number>);
}

Hierarchy (view full)

Implemented by

Properties

allowedSpecial: string[] | "all"
chunkOverlap: number
chunkSize: number
disallowedSpecial: string[] | "all"
encodingName: TiktokenEncoding
keepSeparator: boolean
lengthFunction?: ((text: string) => number) | ((text: string) => Promise<number>)
""