Class TokenStream

Stream which returns and writes data as string tokens into/from file. Stream supports encoding and data buffering.

Inheritance

System.Object

System.MarshalByRefObject

System.IO.Stream

TokenStream

Implements

System.IDisposable

System.Collections.IEnumerable

Inherited Members

System.IO.Stream.Null

System.IO.Stream.CopyToAsync(System.IO.Stream)

System.IO.Stream.CopyToAsync(System.IO.Stream, System.Int32)

System.IO.Stream.CopyToAsync(System.IO.Stream, System.Int32, System.Threading.CancellationToken)

System.IO.Stream.CopyTo(System.IO.Stream)

System.IO.Stream.CopyTo(System.IO.Stream, System.Int32)

System.IO.Stream.Dispose(System.Boolean)

System.IO.Stream.FlushAsync()

System.IO.Stream.FlushAsync(System.Threading.CancellationToken)

System.IO.Stream.CreateWaitHandle()

System.IO.Stream.BeginRead(System.Byte[], System.Int32, System.Int32, System.AsyncCallback, System.Object)

System.IO.Stream.EndRead(System.IAsyncResult)

System.IO.Stream.ReadAsync(System.Byte[], System.Int32, System.Int32)

System.IO.Stream.ReadAsync(System.Byte[], System.Int32, System.Int32, System.Threading.CancellationToken)

System.IO.Stream.BeginWrite(System.Byte[], System.Int32, System.Int32, System.AsyncCallback, System.Object)

System.IO.Stream.EndWrite(System.IAsyncResult)

System.IO.Stream.WriteAsync(System.Byte[], System.Int32, System.Int32)

System.IO.Stream.WriteAsync(System.Byte[], System.Int32, System.Int32, System.Threading.CancellationToken)

System.IO.Stream.ReadByte()

System.IO.Stream.WriteByte(System.Byte)

System.IO.Stream.Synchronized(System.IO.Stream)

System.IO.Stream.ObjectInvariant()

System.IO.Stream.CanTimeout

System.IO.Stream.ReadTimeout

System.IO.Stream.WriteTimeout

System.MarshalByRefObject.MemberwiseClone(System.Boolean)

System.MarshalByRefObject.GetLifetimeService()

System.MarshalByRefObject.InitializeLifetimeService()

System.MarshalByRefObject.CreateObjRef(System.Type)

System.Object.ToString()

System.Object.Equals(System.Object)

System.Object.Equals(System.Object, System.Object)

System.Object.ReferenceEquals(System.Object, System.Object)

System.Object.GetHashCode()

System.Object.GetType()

System.Object.MemberwiseClone()

Namespace: Syncfusion.HTMLUI.Base

Assembly: Syncfusion.HTMLUI.Base.dll

Syntax

public class TokenStream : Stream, IDisposable, IEnumerable

Constructors

TokenStream(Stream)

Constructor. Initializes new object.

Declaration

public TokenStream(Stream input)

Parameters

Type	Name	Description
System.IO.Stream	input	Input data Stream

TokenStream(Stream, Boolean)

Creates Token stream from other stream. Encoding of input stream class is detected automatically.

Declaration

public TokenStream(Stream input, bool controlStream)

Parameters

Type	Name	Description
System.IO.Stream	input	source stream
System.Boolean	controlStream	Indicates whether TokenStream instance must close input stream on own.

TokenStream(Stream, Boolean, Int32)

Constructor. Initializes new object. Encoding of input stream class is detected automatically.

Declaration

public TokenStream(Stream input, bool controlStream, int bufferSize)

Parameters

Type	Name	Description
System.IO.Stream	input	Input data Stream
System.Boolean	controlStream	Indicates whether TokenStream instance must close input stream on own.
System.Int32	bufferSize	Size of the Stream

TokenStream(Stream, Encoding)

Creates Token stream from other stream with user specified encoding.

Declaration

public TokenStream(Stream input, Encoding encoding)

Parameters

Type	Name	Description
System.IO.Stream	input	input stream
System.Text.Encoding	encoding	If encoding of input stream known then you can specify it; otherwise send null value for auto-detection

TokenStream(Stream, Encoding, Boolean)

Creates Token stream from other stream with user specified encoding. Constructor indicates whether to specify class control input stream.

Declaration

public TokenStream(Stream input, Encoding encoding, bool controlStream)

Parameters

Type	Name	Description
System.IO.Stream	input	input stream
System.Text.Encoding	encoding	If encoding of input stream is known then you can specify it; otherwise send null value for auto-detection
System.Boolean	controlStream	Indicates whether TokenStream instance must close input stream on own.

TokenStream(Stream, Encoding, Boolean, Int32)

Constructor. Initializes new object.

Declaration

public TokenStream(Stream input, Encoding encoding, bool controlStream, int bufferSize)

Parameters

Type	Name	Description
System.IO.Stream	input	Input data Stream
System.Text.Encoding	encoding	If encoding of input stream is known then you can specify it; otherwise send null value for auto-detection
System.Boolean	controlStream	Indicates whether TokenStream instance must close input stream on own.
System.Int32	bufferSize	Size of the Stream

TokenStream(String)

Overloaded. Constructor with a file name in open mode.

Declaration

public TokenStream(string fileName)

Parameters

Type	Name	Description
System.String	fileName

TokenStream(String, FileMode)

Constructor allows to control mode of file open operation.

Declaration

public TokenStream(string fileName, FileMode mode)

Parameters

Type	Name	Description
System.String	fileName	File name
System.IO.FileMode	mode	A FileMode constant that determines how to open or create the file.

TokenStream(String, FileMode, FileAccess)

Constructor allows to control File open mode and access flags.

Declaration

public TokenStream(string fileName, FileMode mode, FileAccess access)

Parameters

Type	Name	Description
System.String	fileName	File name
System.IO.FileMode	mode	A FileMode constant that determines how to open or create the file.
System.IO.FileAccess	access	A FileAccess constant that determines how the file can be accessed by the TokenStream object. This gets the CanRead and CanWrite properties of the FileStream object. CanSeek is true if path specifies a disk file

TokenStream(String, FileMode, FileAccess, FileShare)

Constructor allows to override file mode, access and share flags.

Declaration

public TokenStream(string fileName, FileMode mode, FileAccess access, FileShare share)

Parameters

Type	Name	Description
System.String	fileName	File name
System.IO.FileMode	mode	A FileMode constant that determines how to open or create the file.
System.IO.FileAccess	access	A FileAccess constant that determines how the file can be accessed by the TokenStream object. This gets the CanRead and CanWrite properties of the FileStream object. CanSeek is true if path specifies a disk file
System.IO.FileShare	share	A FileShare constant that determines how the file will be shared by processes

TokenStream(String, FileMode, FileAccess, FileShare, Int32)

Open stream. Constructor allows to specify parameter of stream open.

Declaration

public TokenStream(string fileName, FileMode mode, FileAccess access, FileShare share, int bufferSize)

Parameters

Type	Name	Description
System.String	fileName	File name
System.IO.FileMode	mode	A FileMode constant that determines how to open or create the file.
System.IO.FileAccess	access	A FileAccess constant that determines how the file can be accessed by the TokenStream object. This gets the CanRead and CanWrite properties of the FileStream object. CanSeek is true if path specifies a disk file
System.IO.FileShare	share	A FileShare constant that determines how the file will be shared by processes
System.Int32	bufferSize	Size of internal buffer used for read/write operations optimizations

Fields

DEF_BUFFER_SIZE

Stream buffer used for optimizing speed of read/write operations.

Declaration

protected const int DEF_BUFFER_SIZE = 8192

Field Value

Type	Description
System.Int32

DEF_MULTI_TOKEN_SPLIT

Array of atomic tokens which can take more than one symbol.

Declaration

public readonly string[] DEF_MULTI_TOKEN_SPLIT

Field Value

Type	Description
System.String[]

DEF_READ_CHAR_BUFFER_SIZE

Declaration

protected const int DEF_READ_CHAR_BUFFER_SIZE = 128

Field Value

Type	Description
System.Int32

DEF_TOKEN_SPLITS

Array of characters which delimit tokens.

Declaration

public readonly char[] DEF_TOKEN_SPLITS

Field Value

Type	Description
System.Char[]

m_strEndOfLine

New line splitter.

Declaration

protected string m_strEndOfLine

Field Value

Type	Description
System.String

Properties

CanRead

Indicates whether TokenStream can Read.

Declaration

public override bool CanRead { get; }

Property Value

Type	Description
System.Boolean

Overrides

System.IO.Stream.CanRead

CanSeek

Indicates whether TokenStream can Seek.

Declaration

public override bool CanSeek { get; }

Property Value

Type	Description
System.Boolean

Overrides

System.IO.Stream.CanSeek

CanWrite

Indicates whether TokenStream can Write.

Declaration

public override bool CanWrite { get; }

Property Value

Type	Description
System.Boolean

Overrides

System.IO.Stream.CanWrite

CaseSensitive

Indicates whether Multi-character Tokens are case sensitive or not. Default is case sensitive - TRUE. This property greatly reduces performance of TokenStream class an hence used only where it is really needed.

Declaration

public virtual bool CaseSensitive { get; set; }

Property Value

Type	Description
System.Boolean

Encoding

Gets or sets the encoding that is to be used for file reading.

Declaration

public virtual Encoding Encoding { get; set; }

Property Value

Type	Description
System.Text.Encoding

EndLineStyle

Gets or sets End line style ( For different OS ).

Declaration

public virtual NewLineStyle EndLineStyle { get; set; }

Property Value

Type	Description
NewLineStyle

Length

Returns the number of symbols in our Stream.

Declaration

public override long Length { get; }

Property Value

Type	Description
System.Int64

Overrides

System.IO.Stream.Length

MaxMultiTokenLength

Returns the maximum size of multi-symbol token.

Declaration

public virtual int MaxMultiTokenLength { get; }

Property Value

Type	Description
System.Int32

MinMultiTokenLength

Returns the minimum size of multi-symbol token.

Declaration

public virtual int MinMultiTokenLength { get; }

Property Value

Type	Description
System.Int32

MultiCharTokens

Gets or sets multi-symbol atomic tokens. This property can used by top-level abstractions for extracting special lexems, like: ++, --, +=, -=, !=, ==, *=, etc.

Declaration

public virtual string[] MultiCharTokens { get; set; }

Property Value

Type	Description
System.String[]

NewLine

On end of line, string update recalculates lines in stream.

Declaration

public virtual string NewLine { get; set; }

Property Value

Type	Description
System.String

OneCharTokens

Gets or sets an array of characters which can split tokens in stream. Each symbol from this array will be returned by Stream as token with one char, for more complex constructions must use MultiCharTokens property values.

Declaration

public virtual char[] OneCharTokens { get; set; }

Property Value

Type	Description
System.Char[]

Position

Gets or sets position in stream.

Declaration

public override long Position { get; set; }

Property Value

Type	Description
System.Int64

Overrides

System.IO.Stream.Position

Methods

BuildFastCharTree(Hashtable, String[])

Creates a tree of FastChar objects which allows to search faster any word from stream by characters. '\uffff' - symbol indicates end of string from source array.

Declaration

protected virtual void BuildFastCharTree(Hashtable source, string[] value)

Parameters

Type	Name	Description
System.Collections.Hashtable	source	Hash table which will store FastChar's objects.
System.String[]	value	Array of strings.

BuildFastCharTreeEx(Hashtable, String[])

Creates a tree of FastChar objects which allows to search faster any word from stream by characters. '\uffff' - symbol indicates end of string from source array. Alternative method of FastChar tree building.

Declaration

protected virtual void BuildFastCharTreeEx(Hashtable source, string[] value)

Parameters

Type	Name	Description
System.Collections.Hashtable	source	Hash table which will store FastChar's objects.
System.String[]	value	Array of strings.

CalculateLineCount(String)

Calculates and returns the number of lines in stream.

Declaration

protected virtual int CalculateLineCount(string endOfLine)

Parameters

Type	Name	Description
System.String	endOfLine

Returns

Type	Description
System.Int32	Returns total lines count in file.

CalculateLineCount(String, Int64, out Int32)

Calculates the number of lines in stream till specified position.

Declaration

protected virtual int CalculateLineCount(string endOfLine, long tillOffset, out int position)

Parameters

Type	Name	Description
System.String	endOfLine	String which is interpreted as end of line.
System.Int64	tillOffset	Make calculations till specified offset.
System.Int32	position	Returns position part of calculation.

Returns

Type	Description
System.Int32	Number of lines in area from current Position till Offset value.

Close()

Closes current file.

Declaration

public override void Close()

Overrides

System.IO.Stream.Close()

ConvertToLinePosition(Int64, out Int32, out Int32)

Converts offset to Line number and position in line.

Declaration

public virtual void ConvertToLinePosition(long offset, out int line, out int position)

Parameters

Type	Name	Description
System.Int64	offset	Offset which must be converted to line and position values
System.Int32	line	Line part of offset.
System.Int32	position	Position part of offset.

DetectFileEncoding()

Detects file encoding.

Declaration

public virtual Encoding DetectFileEncoding()

Returns

Type	Description
System.Text.Encoding	Detected file encoding by preambles.

DetectFileNewLineStyle()

Checks file and tries to detect which line end code is used in file.

Declaration

public virtual NewLineStyle DetectFileNewLineStyle()

Returns

Type	Description
NewLineStyle	In case of error, NewLineStyle.Windows style is returned.

Dispose()

Clears resources of Token stream class.

Declaration

public void Dispose()

Flush()

Flushes changes from internal buffers to source streams.

Declaration

public override void Flush()

Overrides

System.IO.Stream.Flush()

FromString(String)

Initializes TokenStream from string input.

Declaration

public static TokenStream FromString(string buffer)

Parameters

Type	Name	Description
System.String	buffer	String with input data.

Returns

Type	Description
TokenStream	TokenStream Object

GetEnumerator()

Returns enumerator which gives us stream data by tokens.

Declaration

public virtual IEnumerator GetEnumerator()

Returns

Type	Description
System.Collections.IEnumerator

GetFastChar(Char)

Based on the CaseSensitive property value, returns case sensitive or case insensitive FastChar class instance.

Declaration

protected virtual TokenStream.FastChar GetFastChar(char ch)

Parameters

Type	Name	Description
System.Char	ch	Initializes new instance by character.

Returns

Type	Description
TokenStream.FastChar	Reference on new instance of FastChar class.

GetLinesCount(Int64, Int64)

Calculates the number of lines between two positions in the file.

Declaration

public virtual int GetLinesCount(long startPos, long endPos)

Parameters

Type	Name	Description
System.Int64	startPos	Start position for calculating.
System.Int64	endPos	End position for calculating.

Returns

Type	Description
System.Int32	Count of lines.

GetMultiToken(Char[], out String)

Checks character from array for multi-symbols tokens and returns the quantity of used characters from array for token. It also returns the token. In case, nothing is found, it returns -1. It tries to find the longest token from the list.

Declaration

public virtual int GetMultiToken(char[] ch, out string token)

Parameters

Type	Name	Description
System.Char[]	ch	Characters to check.
System.String	token	Found token.

Returns

Type	Description
System.Int32	Used characters from input array; -1 otherwise.

GetNewLineString(NewLineStyle)

Converts enumeration to its string representation.

Declaration

public virtual string GetNewLineString(NewLineStyle value)

Parameters

Type	Name	Description
NewLineStyle	value	Value to convert.

Returns

Type	Description
System.String	End line string.

IsSplitTokenChar(Char)

Indicates whether character belongs to one symbol tokens separators.

Declaration

public virtual bool IsSplitTokenChar(char ch)

Parameters

Type	Name	Description
System.Char	ch	Character to test

Returns

Type	Description
System.Boolean	True if character belongs to split chars; false otherwise.

Peek()

Peeks one byte from stream.

Declaration

public virtual int Peek()

Returns

Type	Description
System.Int32	-1 if end of stream is reached; one byte from stream otherwise.

PeekChar()

Extracts one char from stream.

Declaration

public virtual int PeekChar()

Returns

Type	Description
System.Int32	-1 if end of stream is reached; integer value of character otherwise.

PeekToken()

Peeks Token from the Stream.

Declaration

public virtual string PeekToken()

Returns

Type	Description
System.String	string.Empty or null if end of stream is reached; extracted token otherwise.

Read()

Reads one byte from stream.

Declaration

public virtual int Read()

Returns

Type	Description
System.Int32	-1 if end of stream is reached; one byte from stream otherwise.

Read(Byte[], Int32, Int32)

Reads data from Stream.

Declaration

public override int Read(byte[] buffer, int offset, int count)

Parameters

Type	Name	Description
System.Byte[]	buffer	Input Stream
System.Int32	offset	Start Position
System.Int32	count	Number of Symbols.

Returns

Type	Description
System.Int32	Number of extracted bytes.

Overrides

System.IO.Stream.Read(System.Byte[], System.Int32, System.Int32)

Read(Char[], Int32, Int32)

Overloaded. Reads data from the Stream.

Declaration

public virtual int Read(char[] buffer, int offset, int count)

Parameters

Type	Name	Description
System.Char[]	buffer	Source Stream
System.Int32	offset	Start position
System.Int32	count	Number of symbols

Returns

Type	Description
System.Int32	number of extracted characters.

ReadChar()

Reads one char from stream and shifts position of stream to next character.

Declaration

public virtual int ReadChar()

Returns

Type	Description
System.Int32	-1 if end of stream; integer value of character otherwise.

ReadLine()

Reads single line.

Declaration

public virtual string ReadLine()

Returns

Type	Description
System.String	String.Empty if end of stream is reached; line from file otherwise.

ReadToken()

Reads one token from stream.

Declaration

public virtual string ReadToken()

Returns

Type	Description
System.String	Null if end of stream is reached; token string otherwise.

ReadTokens(String[], Int32, Int32)

Reads multiple tokens from stream into array.

Declaration

public virtual int ReadTokens(string[] buffer, int position, int length)

Parameters

Type	Name	Description
System.String[]	buffer	Array where extracted tokens must be placed.
System.Int32	position	Start position in the array.
System.Int32	length	Quantity of tokens to extract.

Returns

Type	Description
System.Int32	Quantity of extracted tokens.

Seek(Int64, SeekOrigin)

Sets position within the stream.

Declaration

public override long Seek(long offset, SeekOrigin origin)

Parameters

Type	Name	Description
System.Int64	offset	Start position
System.IO.SeekOrigin	origin	Indicates the reference point that is to be used to obtain the new position.

Returns

Type	Description
System.Int64	New position in stream.

Overrides

System.IO.Stream.Seek(System.Int64, System.IO.SeekOrigin)

SeekOnTokenLength(String, Boolean, SeekOrigin)

Sets the position within the Stream.

Declaration

public virtual long SeekOnTokenLength(string token, bool bBack, SeekOrigin origin)

Parameters

Type	Name	Description
System.String	token	Token
System.Boolean	bBack	Indicates whether to return back.
System.IO.SeekOrigin	origin	Indicates the reference point that is to be used to obtain the new position.

Returns

Type	Description
System.Int64	New position in stream.

SetLength(Int64)

Sets the length of the Stream.

Declaration

public override void SetLength(long value)

Parameters

Type	Name	Description
System.Int64	value	Length of the Stream.

Overrides

System.IO.Stream.SetLength(System.Int64)

SkipLine()

Moves current position in the stream to the next line.

Declaration

public virtual void SkipLine()

Write(Byte[], Int32, Int32)

Writes data into Stream.

Declaration

public override void Write(byte[] buffer, int offset, int count)

Parameters

Type	Name	Description
System.Byte[]	buffer	Data which will be written.
System.Int32	offset	Start Position
System.Int32	count	Number of Symbols

Overrides

System.IO.Stream.Write(System.Byte[], System.Int32, System.Int32)

Implements

System.IDisposable

System.Collections.IEnumerable

Extension Methods

EnumerableExtensions.GetElementType(IEnumerable)

EnumerableExtensions.GetItemPropertyInfo(IEnumerable)

FunctionalExtensions.ForEach<T>(IEnumerable, Action<T>)

FunctionalExtensions.ToList<T>(IEnumerable)

QueryableExtensions.OfQueryable(IEnumerable)

QueryableExtensions.OfQueryable(IEnumerable, Type)

QueryableExtensions.GroupByMany<TElement>(IEnumerable, Type, List<Func<TElement, Object>>)

QueryableExtensions.GroupByMany(IEnumerable, Type, Func<String, Expression>, String[])

QueryableExtensions.GroupByMany(IEnumerable, Type, List<SortDescriptor>, Dictionary<String, IComparer<Object>>, Func<String, Expression>, String[])

QueryableExtensions.GroupByMany(IEnumerable, Type, List<SortDescriptor>, Func<String, Expression>, String[])

ExtensionMethods.GetBytes(Stream)

ExtensionMethods.CopyTo(Stream, Stream)