// --------------------------- // C Headers / Local Headers // --------------------------- #include #include #include #include #include "memory.h" #include "entity.h" #include "token.h" // --------------------------- // makeToken // --------------------------- Token makeToken(void) { Token theToken = { 0 }; theToken.mType = CJ_TT_NONE; theToken.mText = makeCJString(); theToken.mLineNumber = 0; return(theToken); } // --------------------------- // getTokenTypeString // --------------------------- const char *getTokenTypeString(Token *tokenPtr) { const char *type = "Unknown"; switch(tokenPtr->mType) { case CJ_TT_NONE : type = "None"; break; case CJ_TT_LEFT_CURLY : type = "LeftCurly"; break; case CJ_TT_RIGHT_CURLY : type = "RightCurly"; break; case CJ_TT_LEFT_SQUARE : type = "LeftSquare"; break; case CJ_TT_RIGHT_SQUARE: type = "RightSquare"; break; case CJ_TT_COMMA : type = "Comma"; break; case CJ_TT_COLON : type = "Colon"; break; case CJ_TT_TRUE : type = "True"; break; case CJ_TT_FALSE : type = "False"; break; case CJ_TT_NULL : type = "Null"; break; case CJ_TT_STRING : type = "String"; break; case CJ_TT_NUMBER : type = "Number"; break; } return(type); } // --------------------------- // printToken // --------------------------- void printToken(Token *tokenPtr) { const char *type = getTokenTypeString(tokenPtr); printf("(%s) \"%s\"\n", type, tokenPtr->mText.mData); } // --------------------------- // makeToken // --------------------------- TokenList makeTokenList(void) { TokenList theList = { 0, 0, NULL }; return(theList); } // --------------------------- // freeTokenList // --------------------------- void freeTokenList(TokenList *ptr) { Memory.deallocate(ptr->mTokens); ptr->mCount = 0; ptr->mCapacity = 0; ptr->mTokens = NULL; } // --------------------------- // addToTokenList // --------------------------- void addToTokenList(Token *tokenPtr, TokenList *listPtr) { if(listPtr->mCount == listPtr->mCapacity) { if(!listPtr->mCapacity) listPtr->mCapacity = 256; else listPtr->mCapacity *= 2; size_t bytes = listPtr->mCapacity * sizeof(*tokenPtr); if(!listPtr->mTokens) listPtr->mTokens = Memory.allocate(bytes); else listPtr->mTokens = Memory.reallocate(listPtr->mTokens, bytes); Token empty = makeToken(); for(int i = listPtr->mCount + 1; i < listPtr->mCapacity; i++) listPtr->mTokens[i] = empty; } listPtr->mTokens[listPtr->mCount++] = *tokenPtr; } // --------------------------- // startsWithWord // --------------------------- int startsWithWord(const char *s, const char *theWord) { int index = 0; while(s[index] && theWord[index]) { if(s[index] != theWord[index]) return(0); index++; } if(theWord[index]) return(0); return(!isalnum(s[index]) && s[index] != '_'); } // --------------------------- // doStringError // --------------------------- void doStringError( const char *input, int i, int j, int lineNumber, const char *baseMessage, CJString *errMsg ) { size_t n = sizeof(errMsg->mData); char *ptr = errMsg->mData; snprintf(ptr, n, "%s", baseMessage); int length = (j - i); if(length >= 2) { snprintf( ptr + strlen(ptr), n - strlen(ptr), " (%.*s)", length > 5 ? 5 : length, input + i ); } snprintf(ptr + strlen(ptr), n - strlen(ptr), ", line: %d", lineNumber); } // --------------------------- // getTokens // --------------------------- bool getTokens(const char *input, TokenList *ptr, CJString *errMsg) { int lineNumber = 1; int i = 0; bool debug = false; while(input[i]) { if(debug) { printf("i: %d, c: %c\n", i, input[i]); } // Bypass leading whitespace while(input[i] && isspace(input[i])) { if(input[i] == '\n') lineNumber++; i++; } if(!input[i]) break; TokenType theType = CJ_TT_NONE; switch(input[i]) { case '{': theType = CJ_TT_LEFT_CURLY; break; case '}': theType = CJ_TT_RIGHT_CURLY; break; case '[': theType = CJ_TT_LEFT_SQUARE; break; case ']': theType = CJ_TT_RIGHT_SQUARE; break; case ',': theType = CJ_TT_COMMA; break; case ':': theType = CJ_TT_COLON; break; } Token theToken = makeToken(); theToken.mLineNumber = lineNumber; if(theType != CJ_TT_NONE) { theToken.mType = theType; snprintf( theToken.mText.mData, sizeof(theToken.mText.mData), "%c", input[i++] ); } else { char *theWord = NULL; if(startsWithWord(input + i, "true")) { theToken.mType = CJ_TT_TRUE; theWord = "true"; } else if(startsWithWord(input + i, "false")) { theToken.mType = CJ_TT_FALSE; theWord = "false"; } else if(startsWithWord(input + i, "null")) { theToken.mType = CJ_TT_NULL; theWord = "null"; } if(theWord) { snprintf( theToken.mText.mData, sizeof(theToken.mText.mData), "%s", theWord ); i += strlen(theWord); } else if(input[i] == '\'' || input[i] == '"') { int j = i; char end = input[j++]; bool escape = false; bool found = false; while(!found && input[j]) { if(escape) { escape = false; } else { if(input[j] == '\\') { escape = true; } else if(input[j] == '\n') { if(j > 0 && input[j - 1] == '\r') --j; doStringError( input, i, j, lineNumber, "Newline found within string", errMsg ); freeTokenList(ptr); return(false); } else if(input[j] == end) { found = true; } } j++; } if(!found) { doStringError( input, i, j, lineNumber, "Unterminated string", errMsg ); freeTokenList(ptr); return(false); } int length = (j - i) - 2; if(length >= CJ_STRING_LENGTH) { doStringError(input, i, j, lineNumber, "Overlong string", errMsg); freeTokenList(ptr); return(false); } theToken.mType = CJ_TT_STRING; snprintf( theToken.mText.mData, sizeof(theToken.mText.mData), "%.*s", length, input + i + 1 ); i = j; } else if(isdigit(input[i]) || input[i] == '.' || input[i] == '-') { int nDots = (input[i] == '.'); bool valid = true; int j = i + 1; int digits = 0; while(valid && input[j]) { if(input[j] == '.') { if(++nDots > 1) valid = false; } else if(!isdigit(input[j])) { if(isalpha(input[j]) || input[j] == '_') valid = false; else break; } else digits++; j++; } if(!valid || digits == 0) { snprintf( errMsg->mData, sizeof(errMsg->mData), "Invalid number: (%.*s), line: %d", j - i, input + i, lineNumber ); freeTokenList(ptr); return(false); } theToken.mType = CJ_TT_NUMBER; snprintf( theToken.mText.mData, sizeof(theToken.mText.mData), "%.*s", j - i, input + i ); i = j; } else { snprintf( errMsg->mData, sizeof(errMsg->mData), "Invalid token beginning with: (%.*s...), line: %d", 7, input + i, lineNumber ); freeTokenList(ptr); return(false); } } // Add the new token addToTokenList(&theToken, ptr); } return(true); }