requires: {#ReadStream}.
provides: {#Tokenizer}.

prototypes addPrototype: #Tokenizer derivedFrom: {StreamProcessor}.
"A stream that collects characters and emits words separated by
any of the given separators."

Tokenizer addSlot: #separators valued: ASCIICharacter Whitespace copy.

s@(Tokenizer traits) on: target
[
  s source: target.
  s
].

s@(Tokenizer traits) newOn: target withSeparators: separators
[| result |
  result: (s newOn: target).
  result separators: (separators as: result separators).
  result
].

s@(ReadStream traits) split
"Answer a Tokenizer over the input argument with the default separators."
[Tokenizer newOn: s].

s@(ReadStream traits) splitWith: separators
"Answer a Tokenizer over the input argument with the given separators."
[Tokenizer newOn: s withSeparators: separators].

s@(Tokenizer traits) skipSeparators
[
  [s isSeparator: s source peek]
    whileTrue: [s source next].
  s
].

s@(Tokenizer traits) atEnd
[
  [s skipSeparators] on: Exhaustion do: [| :c | ^ True].
  s source atEnd
].

s@(Tokenizer traits) reset
[
  resend.
  s source reset.
  s
].

s@(Tokenizer traits) isSeparator: char
[
  s separators includes: char
].

s@(Tokenizer traits) next
[| result |
  result: s source arrayType newEmpty writer.
  [[s skipSeparators.
    [result nextPut: s source next.
      s isSeparator: s source peek]
      whileFalse]
      breakOn: Exhaustion]
    ensure: [result position isZero ifTrue: [s exhausted]].
   result contents
].