シェブロテインの学習

https://github.com/witheve/Eve/blob/master/src/parser/parser.ts

import * as chev from "chevrotain";

var {Lexer, tokenMatcher} = chev;
export var Token = chev.Token;

/* .... */

// Comments
export class CommentLine extends Token { static PATTERN = /\/\/.*\n/; label = "comment"; static GROUP = "comments"; }

// Operators
export class Equality extends Token { static PATTERN = /:|=/; label = "equality"; }
export class Comparison extends Token { static PATTERN = />=|<=|!=|>|</; label = "comparison"; }
export class AddInfix extends Token { static PATTERN = /\+|-/; label = "infix"; }
export class MultInfix extends Token { static PATTERN = /\*|\//; label = "infix"; }
export class Merge extends Token { static PATTERN = /<-/; label = "merge"; }
export class Set extends Token { static PATTERN = /:=/; label = "set"; }
export class Mutate extends Token { static PATTERN = /\+=|-=/; label = "mutate"; }
export class Dot extends Token { static PATTERN = /\./; label = "dot"; }
export class Pipe extends Token { static PATTERN = /\|/; label = "pipe"; }

↑TypeScriptで、トークン定義をクラス定義でしているようだ。chevrotain.d.ts を見るとITokenインターフェイス定義はあった↓

export interface IToken {
  /** The textual representation of the Token as it appeared in the text. */
  image: string

  /** Offset of the first character of the Token. 0-indexed. */
  startOffset: number

  /** Line of the first character of the Token. 1-indexed. */
  startLine?: number

  /**
   * Column of the first character of the Token. 1-indexed.
   *
   * For token foo in the following line, startColumn will be 3 and endColumn will be 5.
   * ```
   * a foo
   * 123456
   * ```
   */
  startColumn?: number

  /**
   * Offset of the last character of the Token. 0-indexed.
   * Note that this points at the last character, not the end of the token, so the original image would be
   * `input.substring(token.startOffset, token.endOffset + 1)`.
   */
  endOffset?: number

  /** Line of the last character of the Token. 1-indexed. Will be the same as startLine for single-line tokens.*/
  endLine?: number

  /** Column of the last character of the Token. 1-indexed. See also startColumn. */
  endColumn?: number

  /** this marks if a Token does not really exist and has been inserted "artificially" during parsing in rule error recovery. */
  isInsertedInRecovery?: boolean

  /** An number index representing the type of the Token use <getTokenConstructor> to get the Token Type from a token "instance"  */
  tokenTypeIdx: number

  /**
   * The actual Token Type of this Token "instance"
   * This is the same Object returned by the "createToken" API.
   * This property is very useful for debugging the Lexing and Parsing phases.
   */
  tokenType: TokenType

  /**
   * Custom Payload value, this is an optional feature of Custom Token Patterns
   * For additional details see the docs:
   * https://chevrotain.io/docs/guide/custom_token_patterns.html#custom-payloads
   */
  payload?: any
}

static PATTERN でいいのか？ labelフィールドもないし？ なんか奇妙だ。どうも今はTokenというクラスはなくなり、TokenTypeというインターフェイスだけがあるようだ。

export interface TokenType {
  name: string
  GROUP?: string
  PATTERN?: TokenPattern
  LABEL?: string
  LONGER_ALT?: TokenType | TokenType[]
  POP_MODE?: boolean
  PUSH_MODE?: string
  LINE_BREAKS?: boolean
  CATEGORIES?: TokenType[]
  tokenTypeIdx?: number
  categoryMatches?: number[]
  categoryMatchesMap?: {
    [tokType: number]: boolean
  }
  isParent?: boolean
  START_CHARS_HINT?: (string | number)[]
}

このインターフェイスを満たすクラスのオブジェクトを作れば確かにトークン定義にはなるが、createToken関数のほうが簡単な気がする。