324 lines
9.4 KiB
C
324 lines
9.4 KiB
C
/*
|
|
AngelCode Scripting Library
|
|
Copyright (c) 2003-2019 Andreas Jonsson
|
|
|
|
This software is provided 'as-is', without any express or implied
|
|
warranty. In no event will the authors be held liable for any
|
|
damages arising from the use of this software.
|
|
|
|
Permission is granted to anyone to use this software for any
|
|
purpose, including commercial applications, and to alter it and
|
|
redistribute it freely, subject to the following restrictions:
|
|
|
|
1. The origin of this software must not be misrepresented; you
|
|
must not claim that you wrote the original software. If you use
|
|
this software in a product, an acknowledgment in the product
|
|
documentation would be appreciated but is not required.
|
|
|
|
2. Altered source versions must be plainly marked as such, and
|
|
must not be misrepresented as being the original software.
|
|
|
|
3. This notice may not be removed or altered from any source
|
|
distribution.
|
|
|
|
The original version of this library can be located at:
|
|
http://www.angelcode.com/angelscript/
|
|
|
|
Andreas Jonsson
|
|
andreas@angelcode.com
|
|
*/
|
|
|
|
|
|
//
|
|
// as_tokendef.h
|
|
//
|
|
// Definitions for tokens identifiable by the tokenizer
|
|
//
|
|
|
|
|
|
#ifndef AS_TOKENDEF_H
|
|
#define AS_TOKENDEF_H
|
|
|
|
#include "as_config.h"
|
|
|
|
BEGIN_AS_NAMESPACE
|
|
|
|
enum eTokenType {
|
|
ttUnrecognizedToken,
|
|
|
|
ttEnd, // End of file
|
|
|
|
// White space and comments
|
|
ttWhiteSpace, // ' ', '\t', '\r', '\n', UTF8 byte-order-mark
|
|
ttOnelineComment, // // \n
|
|
ttMultilineComment, // /* */
|
|
|
|
// Atoms
|
|
ttIdentifier, // abc123
|
|
ttIntConstant, // 1234
|
|
ttFloatConstant, // 12.34e56f
|
|
ttDoubleConstant, // 12.34e56
|
|
ttStringConstant, // "123"
|
|
ttMultilineStringConstant, //
|
|
ttHeredocStringConstant, // """text"""
|
|
ttNonTerminatedStringConstant, // "123
|
|
ttBitsConstant, // 0xFFFF
|
|
|
|
// Math operators
|
|
ttPlus, // +
|
|
ttMinus, // -
|
|
ttStar, // *
|
|
ttSlash, // /
|
|
ttPercent, // %
|
|
ttStarStar, // **
|
|
|
|
ttHandle, // @
|
|
|
|
ttAddAssign, // +=
|
|
ttSubAssign, // -=
|
|
ttMulAssign, // *=
|
|
ttDivAssign, // /=
|
|
ttModAssign, // %=
|
|
ttPowAssign, // **=
|
|
|
|
ttOrAssign, // |=
|
|
ttAndAssign, // &=
|
|
ttXorAssign, // ^=
|
|
ttShiftLeftAssign, // <<=
|
|
ttShiftRightLAssign, // >>=
|
|
ttShiftRightAAssign, // >>>=
|
|
|
|
ttInc, // ++
|
|
ttDec, // --
|
|
|
|
ttDot, // .
|
|
ttScope, // ::
|
|
|
|
// Statement tokens
|
|
ttAssignment, // =
|
|
ttEndStatement, // ;
|
|
ttListSeparator, // ,
|
|
ttStartStatementBlock, // {
|
|
ttEndStatementBlock, // }
|
|
ttOpenParanthesis, // (
|
|
ttCloseParanthesis, // )
|
|
ttOpenBracket, // [
|
|
ttCloseBracket, // ]
|
|
ttAmp, // &
|
|
|
|
// Bitwise operators
|
|
ttBitOr, // |
|
|
ttBitNot, // ~
|
|
ttBitXor, // ^
|
|
ttBitShiftLeft, // <<
|
|
ttBitShiftRight, // >> // TODO: In Java this is the arithmetical shift
|
|
ttBitShiftRightArith, // >>> // TODO: In Java this is the logical shift
|
|
|
|
// Compare operators
|
|
ttEqual, // ==
|
|
ttNotEqual, // !=
|
|
ttLessThan, // <
|
|
ttGreaterThan, // >
|
|
ttLessThanOrEqual, // <=
|
|
ttGreaterThanOrEqual, // >=
|
|
|
|
ttQuestion, // ?
|
|
ttColon, // :
|
|
|
|
// Reserved keywords
|
|
ttIf, // if
|
|
ttElse, // else
|
|
ttFor, // for
|
|
ttWhile, // while
|
|
ttBool, // bool
|
|
ttFuncDef, // funcdef
|
|
ttImport, // import
|
|
ttInt, // int
|
|
ttInt8, // int8
|
|
ttInt16, // int16
|
|
ttInt64, // int64
|
|
ttInterface, // interface
|
|
ttIs, // is
|
|
ttNotIs, // !is
|
|
ttUInt, // uint
|
|
ttUInt8, // uint8
|
|
ttUInt16, // uint16
|
|
ttUInt64, // uint64
|
|
ttFloat, // float
|
|
ttVoid, // void
|
|
ttTrue, // true
|
|
ttFalse, // false
|
|
ttReturn, // return
|
|
ttNot, // not
|
|
ttAnd, // and, &&
|
|
ttOr, // or, ||
|
|
ttXor, // xor, ^^
|
|
ttBreak, // break
|
|
ttContinue, // continue
|
|
ttConst, // const
|
|
ttDo, // do
|
|
ttDouble, // double
|
|
ttSwitch, // switch
|
|
ttCase, // case
|
|
ttDefault, // default
|
|
ttIn, // in
|
|
ttOut, // out
|
|
ttInOut, // inout
|
|
ttNull, // null
|
|
ttClass, // class
|
|
ttTypedef, // typedef
|
|
ttEnum, // enum
|
|
ttCast, // cast
|
|
ttPrivate, // private
|
|
ttProtected, // protected
|
|
ttNamespace, // namespace
|
|
ttMixin, // mixin
|
|
ttAuto, // auto
|
|
ttTry, // try
|
|
ttCatch // catch
|
|
};
|
|
|
|
struct sTokenWord {
|
|
const char *word;
|
|
size_t wordLength;
|
|
eTokenType tokenType;
|
|
};
|
|
|
|
#define asTokenDef(str, tok) {str, sizeof(str)-1, tok}
|
|
|
|
sTokenWord const tokenWords[] = {
|
|
asTokenDef("+", ttPlus),
|
|
asTokenDef("+=", ttAddAssign),
|
|
asTokenDef("++", ttInc),
|
|
asTokenDef("-", ttMinus),
|
|
asTokenDef("-=", ttSubAssign),
|
|
asTokenDef("--", ttDec),
|
|
asTokenDef("*", ttStar),
|
|
asTokenDef("*=", ttMulAssign),
|
|
asTokenDef("/", ttSlash),
|
|
asTokenDef("/=", ttDivAssign),
|
|
asTokenDef("%", ttPercent),
|
|
asTokenDef("%=", ttModAssign),
|
|
asTokenDef("**", ttStarStar),
|
|
asTokenDef("**=", ttPowAssign),
|
|
asTokenDef("=", ttAssignment),
|
|
asTokenDef("==", ttEqual),
|
|
asTokenDef(".", ttDot),
|
|
asTokenDef("|", ttBitOr),
|
|
asTokenDef("|=", ttOrAssign),
|
|
asTokenDef("||", ttOr),
|
|
asTokenDef("&", ttAmp),
|
|
asTokenDef("&=", ttAndAssign),
|
|
asTokenDef("&&", ttAnd),
|
|
asTokenDef("^", ttBitXor),
|
|
asTokenDef("^=", ttXorAssign),
|
|
asTokenDef("^^", ttXor),
|
|
asTokenDef("<", ttLessThan),
|
|
asTokenDef("<=", ttLessThanOrEqual),
|
|
asTokenDef("<<", ttBitShiftLeft),
|
|
asTokenDef("<<=", ttShiftLeftAssign),
|
|
asTokenDef(">", ttGreaterThan),
|
|
asTokenDef(">=", ttGreaterThanOrEqual),
|
|
asTokenDef(">>", ttBitShiftRight),
|
|
asTokenDef(">>=", ttShiftRightLAssign),
|
|
asTokenDef(">>>", ttBitShiftRightArith),
|
|
asTokenDef(">>>=", ttShiftRightAAssign),
|
|
asTokenDef("~", ttBitNot),
|
|
asTokenDef(";", ttEndStatement),
|
|
asTokenDef(",", ttListSeparator),
|
|
asTokenDef("{", ttStartStatementBlock),
|
|
asTokenDef("}", ttEndStatementBlock),
|
|
asTokenDef("(", ttOpenParanthesis),
|
|
asTokenDef(")", ttCloseParanthesis),
|
|
asTokenDef("[", ttOpenBracket),
|
|
asTokenDef("]", ttCloseBracket),
|
|
asTokenDef("?", ttQuestion),
|
|
asTokenDef(":", ttColon),
|
|
asTokenDef("::", ttScope),
|
|
asTokenDef("!", ttNot),
|
|
asTokenDef("!=", ttNotEqual),
|
|
asTokenDef("!is", ttNotIs),
|
|
asTokenDef("@", ttHandle),
|
|
asTokenDef("and", ttAnd),
|
|
asTokenDef("auto", ttAuto),
|
|
asTokenDef("bool", ttBool),
|
|
asTokenDef("break", ttBreak),
|
|
asTokenDef("case", ttCase),
|
|
asTokenDef("cast", ttCast),
|
|
asTokenDef("catch", ttCatch),
|
|
asTokenDef("class", ttClass),
|
|
asTokenDef("const", ttConst),
|
|
asTokenDef("continue", ttContinue),
|
|
asTokenDef("default", ttDefault),
|
|
asTokenDef("do", ttDo),
|
|
#ifdef AS_USE_DOUBLE_AS_FLOAT
|
|
asTokenDef("double", ttFloat),
|
|
#else
|
|
asTokenDef("double", ttDouble),
|
|
#endif
|
|
asTokenDef("else", ttElse),
|
|
asTokenDef("enum", ttEnum),
|
|
asTokenDef("false", ttFalse),
|
|
asTokenDef("float", ttFloat),
|
|
asTokenDef("for", ttFor),
|
|
asTokenDef("funcdef", ttFuncDef),
|
|
asTokenDef("if", ttIf),
|
|
asTokenDef("import", ttImport),
|
|
asTokenDef("in", ttIn),
|
|
asTokenDef("inout", ttInOut),
|
|
asTokenDef("int", ttInt),
|
|
asTokenDef("int8", ttInt8),
|
|
asTokenDef("int16", ttInt16),
|
|
asTokenDef("int32", ttInt),
|
|
asTokenDef("int64", ttInt64),
|
|
asTokenDef("interface", ttInterface),
|
|
asTokenDef("is", ttIs),
|
|
asTokenDef("mixin", ttMixin),
|
|
asTokenDef("namespace", ttNamespace),
|
|
asTokenDef("not", ttNot),
|
|
asTokenDef("null", ttNull),
|
|
asTokenDef("or", ttOr),
|
|
asTokenDef("out", ttOut),
|
|
asTokenDef("private", ttPrivate),
|
|
asTokenDef("protected", ttProtected),
|
|
asTokenDef("return", ttReturn),
|
|
asTokenDef("switch", ttSwitch),
|
|
asTokenDef("true", ttTrue),
|
|
asTokenDef("try", ttTry),
|
|
asTokenDef("typedef", ttTypedef),
|
|
asTokenDef("uint", ttUInt),
|
|
asTokenDef("uint8", ttUInt8),
|
|
asTokenDef("uint16", ttUInt16),
|
|
asTokenDef("uint32", ttUInt),
|
|
asTokenDef("uint64", ttUInt64),
|
|
asTokenDef("void", ttVoid),
|
|
asTokenDef("while", ttWhile),
|
|
asTokenDef("xor", ttXor),
|
|
};
|
|
|
|
const unsigned int numTokenWords = sizeof(tokenWords) / sizeof(sTokenWord);
|
|
|
|
const char *const whiteSpace = " \t\r\n";
|
|
|
|
// Some keywords that are not considered tokens by the parser
|
|
// These only have meaning in specific situations. Outside these
|
|
// situations they are treated as normal identifiers.
|
|
const char *const THIS_TOKEN = "this";
|
|
const char *const FROM_TOKEN = "from";
|
|
const char *const SUPER_TOKEN = "super";
|
|
const char *const SHARED_TOKEN = "shared";
|
|
const char *const FINAL_TOKEN = "final";
|
|
const char *const OVERRIDE_TOKEN = "override";
|
|
const char *const GET_TOKEN = "get";
|
|
const char *const SET_TOKEN = "set";
|
|
const char *const ABSTRACT_TOKEN = "abstract";
|
|
const char *const FUNCTION_TOKEN = "function";
|
|
const char *const IF_HANDLE_TOKEN = "if_handle_then_const";
|
|
const char *const EXTERNAL_TOKEN = "external";
|
|
const char *const EXPLICIT_TOKEN = "explicit";
|
|
const char *const PROPERTY_TOKEN = "property";
|
|
|
|
END_AS_NAMESPACE
|
|
|
|
#endif
|