package lexer

import "github.com/azin-lang/Azin/pkg/lexer"

Index

• type Lexer
  • func New(file *source.File, diag *diagnostics.Engine) *Lexer
  • func (l *Lexer) Tokenize() []token2.Token
  • func (l *Lexer) Tokens() iter.Seq[token2.Token]

Types

type Lexer

type Lexer struct {
	// contains filtered or unexported fields
}

Lexer performs lexical analysis on a source file, transforming raw text into a stream of syntax tokens.

func New

func New(file *source.File, diag *diagnostics.Engine) *Lexer

New initializes a new Lexer for the given source file.

func (*Lexer) Tokenize

func (l *Lexer) Tokenize() []token2.Token

Tokenize eagerly scans the entire file and returns a slice of all tokens.

func (*Lexer) Tokens

func (l *Lexer) Tokens() iter.Seq[token2.Token]

Tokens returns an iterator over the tokens in the source file. It yields tokens lazily until the end of the file is reached.