package token

import "github.com/azin-lang/Azin/internal/token"

Package token defines constants and types representing Azin lexical tokens.

Index

Variables

var Keywords = map[string]Kind{
	"fn":     KwFn,
	"do":     KwDo,
	"var":    KwVar,
	"return": KwReturn,
	"end":    KwEnd,
	"char":   KwChar,
	"int":    KwInt,
	"str":    KwString,
	"float":  KwFloat,
	"if":     KwIf,
	"then":   KwThen,
}

Keywords maps string literals to keyword token kinds.

Types

type Kind

type Kind uint8

Kind is the set of lexical token types. 

const (
	Unknown Kind = iota // unknown

	Identifier       // identifier
	IntegerLiteral   // integer_literal
	StringLiteral    // string_literal
	FloatLiteral     // float_literal
	CharacterLiteral // character_literal

	KwFn     // kw_fn
	KwDo     // kw_do
	KwVar    // kw_var
	KwReturn // kw_return
	KwEnd    // kw_end
	KwChar   // kw_char
	KwInt    // kw_int
	KwString // kw_string
	KwFloat  // kw_float
	KwIf     // kw_if
	KwThen   // kw_then

	Plus           // plus
	Minus          // minus
	Star           // star
	Slash          // slash
	Equal          // equal
	EqualEqual     // equal_equal
	Bang           // bang
	BangEqual      // bang_equal
	Less           // less
	LessEqual      // less_equal
	Greater        // greater
	GreaterEqual   // greater_equal
	Arrow          // arrow
	Modulo         // modulo
	Pipe           // pipe
	LogicalOr      // logical_or
	LogicalAnd     // logical_and
	Ampersand      // ampersand
	Caret          // caret
	Tilde          // tilde
	PlusEqual      // plus_equal
	MinusEqual     // minus_equal
	StarEqual      // star_equal
	SlashEqual     // slash_equal
	ModuloEqual    // modulo_equal
	CaretEqual     // caret_equal
	PipeEqual      // pipe_equal
	AmpersandEqual // ampersand_equal
	PlusPlus       // plus_plus
	MinusMinus     // minus_minus
	LessLess       // less_less
	GreaterGreater // greater_greater

	LeftParen    // left_paren
	RightParen   // right_paren
	LeftBrace    // left_brace
	RightBrace   // right_brace
	Comma        // comma
	Semicolon    // semicolon
	Colon        // colon
	Dot          // dot
	LeftBracket  // left_bracket
	RightBracket // right_bracket

	EOF // eof

	Error // error
)
func (Kind) HasText
func (k Kind) HasText() bool

HasText reports whether the token kind contains variable text content.

func (Kind) String
func (i Kind) String() string

type Position

type Position struct {
	Offset uint32
}

Position represents a byte offset within a source file.

type Token

type Token struct {
	Kind     Kind
	Position Position
	Length   uint32
}

Token represents a single lexical unit.