lisp::Lexer Class Reference

#include <lexer.hpp>

List of all members.

Public Types
enum	TokenType {   TOKEN_EOF, TOKEN_OPEN_PAREN, TOKEN_CLOSE_PAREN, TOKEN_SYMBOL,   TOKEN_STRING, TOKEN_INTEGER, TOKEN_REAL, TOKEN_TRUE,   TOKEN_FALSE }
Public Member Functions
	Lexer (std::istream &stream)
	~Lexer ()
TokenType	getNextToken ()
const char *	getString () const
int	getLineNumber () const
Private Types
enum	{ MAX_TOKEN_LENGTH = 16384, BUFFER_SIZE = 1024 }
Private Member Functions
void	nextChar ()
void	addChar ()
	Lexer (const Lexer &)
Lexer &	operator= (const Lexer &)
Private Attributes
std::istream &	stream
bool	eof
int	linenumber
char	buffer [BUFFER_SIZE+1]
char *	bufend
char *	bufpos
int	c
char	token_string [MAX_TOKEN_LENGTH+1]
int	token_length

---

Detailed Description

Definition at line 24 of file lexer.hpp.

---

Member Enumeration Documentation

enum lisp::Lexer::TokenType

Enumerator:

TOKEN_EOF
TOKEN_OPEN_PAREN
TOKEN_CLOSE_PAREN
TOKEN_SYMBOL
TOKEN_STRING
TOKEN_INTEGER
TOKEN_REAL
TOKEN_TRUE
TOKEN_FALSE

Definition at line 27 of file lexer.hpp.

                 {
    TOKEN_EOF,
    TOKEN_OPEN_PAREN,
    TOKEN_CLOSE_PAREN,
    TOKEN_SYMBOL,
    TOKEN_STRING,
    TOKEN_INTEGER,
    TOKEN_REAL,
    TOKEN_TRUE,
    TOKEN_FALSE
  };

anonymous enum [private]

Enumerator:

MAX_TOKEN_LENGTH
BUFFER_SIZE

Definition at line 49 of file lexer.hpp.

       {
    MAX_TOKEN_LENGTH = 16384,
    BUFFER_SIZE = 1024
  };

---

Constructor & Destructor Documentation

lisp::Lexer::Lexer

(

std::istream &

stream

)

Definition at line 26 of file lexer.cpp.

References bufend, bufpos, and nextChar().

                                  :
  stream(newstream), 
  eof(false), 
  linenumber(0),
  bufend(),
  bufpos(),
  c(),
  token_length()
{
  // trigger a refill of the buffer
  bufpos = NULL;
  bufend = NULL;
  nextChar();
}

lisp::Lexer::~Lexer

(

)

Definition at line 41 of file lexer.cpp.

{
}

lisp::Lexer::Lexer

(

const Lexer &

)

[private]

---

Member Function Documentation

Lexer::TokenType lisp::Lexer::getNextToken

(

)

Definition at line 82 of file lexer.cpp.

References addChar(), c, linenumber, MAX_TOKEN_LENGTH, nextChar(), TOKEN_CLOSE_PAREN, TOKEN_EOF, TOKEN_FALSE, TOKEN_INTEGER, token_length, TOKEN_OPEN_PAREN, TOKEN_REAL, TOKEN_STRING, token_string, TOKEN_SYMBOL, and TOKEN_TRUE.

Referenced by lisp::Parser::parse(), and lisp::Parser::read().

{
  static const char* delims = "\"();";

  while(isspace(c)) {
    nextChar();
  }

  token_length = 0;

  switch(c) {
    case ';': // comment
      while(c != '\n') {
        nextChar();
      }
      return getNextToken(); // and again
    case '(':
      nextChar();
      return TOKEN_OPEN_PAREN;
    case ')':
      nextChar();
      return TOKEN_CLOSE_PAREN;
    case '"': {  // string
      int startline = linenumber;
      while(1) {
        nextChar();
        switch(c) {
          case '"':
            nextChar();
            goto string_finished;
          case '\r':
            continue;
          case '\n':
            break;
          case '\\':
            nextChar();
            switch(c) {
              case 'n':
                c = '\n';
                break;
              case 't':
                c = '\t';
                break;
            }
            break;
          case EOF: {
            std::stringstream msg;
            msg << "Parse error in line " << startline << ": "
                << "EOF while parsing string.";
            throw std::runtime_error(msg.str());
          }
          default:
            break;
        }
        if(token_length < MAX_TOKEN_LENGTH)
          token_string[token_length++] = c;
      }
      string_finished:
      token_string[token_length] = 0;
      return TOKEN_STRING;
    }
    case '#': // constant
      nextChar();

      while(isalnum(c) || c == '_') {
        addChar();
      }
      token_string[token_length] = 0;

      if(strcmp(token_string, "t") == 0)
        return TOKEN_TRUE;
      if(strcmp(token_string, "f") == 0)
        return TOKEN_FALSE;

      // we only handle #t and #f constants at the moment...
      {
        std::stringstream msg;
        msg << "Parse Error in line " << linenumber << ": "
            << "Unknown constant '" << token_string << "'.";
        throw std::runtime_error(msg.str());
      }

    case EOF:
      return TOKEN_EOF;

    default:
      if(isdigit(c) || c == '-') {
        bool have_nondigits = false;
        bool have_digits = false;
        int have_floating_point = 0;

        do {
          if(isdigit(c))
            have_digits = true;
          else if(c == '.')
            ++have_floating_point;
          else if(isalnum(c) || c == '_')
            have_nondigits = true;

          addChar();
        } while(!isspace(c) && !strchr(delims, c));

        token_string[token_length] = 0;

        // no nextChar

        if(have_nondigits || !have_digits || have_floating_point > 1)
          return TOKEN_SYMBOL;
        else if(have_floating_point == 1)
          return TOKEN_REAL;
        else
          return TOKEN_INTEGER;
      } else {
        do {
          addChar();
        } while(!isspace(c) && !strchr(delims, c));
        token_string[token_length] = 0;

        // no nextChar

        return TOKEN_SYMBOL;
      }
  }
}

const char* lisp::Lexer::getString

(

)

const [inline]

Definition at line 43 of file lexer.hpp.

References token_string.

Referenced by lisp::Parser::read().

  { return token_string; }

int lisp::Lexer::getLineNumber

(

)

const [inline]

Definition at line 45 of file lexer.hpp.

References linenumber.

Referenced by lisp::Parser::parse_error().

  { return linenumber; }

void lisp::Lexer::nextChar

(

)

[inline, private]

Definition at line 46 of file lexer.cpp.

References bufend, buffer, BUFFER_SIZE, bufpos, c, eof, linenumber, and stream.

Referenced by addChar(), getNextToken(), and Lexer().

{
  if(bufpos >= bufend) {
    if(eof) {
      c = EOF;
      return;
    }
    stream.read(buffer, BUFFER_SIZE);
    size_t bytes_read = stream.gcount();

    bufpos = buffer;
    bufend = buffer + bytes_read;

    // the following is a hack that appends an additional ' ' at the end of
    // the file to avoid problems when parsing symbols/elements and a sudden
    // EOF. This is faster than relying on unget and IMO also nicer.
    if(bytes_read == 0 || stream.eof()) {
      eof = true;
      *bufend = ' ';
      ++bufend;
    }
  }
  c = *bufpos++;
  if(c == '\n')
    ++linenumber;
}

void lisp::Lexer::addChar

(

)

[inline, private]

Definition at line 74 of file lexer.cpp.

References c, MAX_TOKEN_LENGTH, nextChar(), token_length, and token_string.

Referenced by getNextToken().

{
  if(token_length < MAX_TOKEN_LENGTH)
    token_string[token_length++] = c;
  nextChar();
}

Lexer& lisp::Lexer::operator=

(

const Lexer &

)

[private]

---

Member Data Documentation

std::istream& lisp::Lexer::stream [private]

Definition at line 58 of file lexer.hpp.

Referenced by nextChar().

bool lisp::Lexer::eof [private]

Definition at line 59 of file lexer.hpp.

Referenced by nextChar().

int lisp::Lexer::linenumber [private]

Definition at line 60 of file lexer.hpp.

Referenced by getLineNumber(), getNextToken(), and nextChar().

char lisp::Lexer::buffer[BUFFER_SIZE+1] [private]

Definition at line 61 of file lexer.hpp.

Referenced by nextChar().

char* lisp::Lexer::bufend [private]

Definition at line 62 of file lexer.hpp.

Referenced by Lexer(), and nextChar().

char* lisp::Lexer::bufpos [private]

Definition at line 63 of file lexer.hpp.

Referenced by Lexer(), and nextChar().

int lisp::Lexer::c [private]

Definition at line 64 of file lexer.hpp.

Referenced by addChar(), getNextToken(), and nextChar().

char lisp::Lexer::token_string[MAX_TOKEN_LENGTH+1] [private]

Definition at line 65 of file lexer.hpp.

Referenced by addChar(), getNextToken(), and getString().

int lisp::Lexer::token_length [private]

Definition at line 66 of file lexer.hpp.

Referenced by addChar(), and getNextToken().

---

The documentation for this class was generated from the following files:

• src/lisp/lexer.hpp
• src/lisp/lexer.cpp

---