Java 文件标记器错误
所以我编写了这个程序来标记一个包含代码的文件，并将每个标识符、关键字、数字和符号分离成一个数字标记。我遇到的问题是，虽然它将为每行中的第一个单词分配正确的标记。后一行中的每个标记都被视为标识符。这是我的标记器函数，我认为问题在于：Java 文件标记器错误,java,tokenize,Java,Tokenize,所以我编写了这个程序来标记一个包含代码的文件，并将每个标识符、关键字、数字和符号分离成一个数字标记。我遇到的问题是，虽然它将为每行中的第一个单词分配正确的标记。后一行中的每个标记都被视为标识符。这是我的标记器函数，我认为问题在于： public class cmmLex { public static boolean isLetter(char b){ char[] letters_ = {'a','b','c','d','e','f','g',
public class cmmLex {
public static boolean isLetter(char b){
char[] letters_ = {'a','b','c','d','e','f','g','h','i','j','k','l','m','n','o','p','q','r','s','t','u','v','w','x','y','z','A','B','C','D',
'E','F','G','H','I','J','K','L','M','N','O','P','Q','R','S','T','U','V','W','X','Y','Z','_'};
for (int i = 0; i < letters_.length; i++) {
if(b == letters_[i])
return true;
}
return false;
}
public static boolean isNumber(char b){
char[] numbers = {'0','1','2','3','4','5','6','7','8','9'};
for (int i = 0; i < numbers.length; i++) {
if(b == numbers[i])
return true;
}
return false;
}
public static boolean isKeyword(StringBuffer str){
String[] keywords = {"int", "double", "if", "while","return","void","else"};
for (int i = 0; i < keywords.length; i++) {
if (keywords[i].equals(str.toString()))
return true;
}
return false;
}
public static boolean isSymbol(char a){
char[] symbols = {'+','-','*','/','<','>','!','=',',','.','(',')','[',']','{','}'};
for (int i = 0; i < symbols.length; i++) {
if(a == symbols[i])
return true;
}
return false;
}
public static void lexMe(StringBuffer string)
{
if(isKeyword(string)){
switch(string.toString()){
case "double":
System.out.print("0 ");
break;
case "else":
System.out.print("1 ");
break;
case "if":
System.out.print("2 ");
break;
case "int":
System.out.print("3 ");
break;
case "return":
System.out.print("4 ");
break;
case "void":
System.out.print("5 ");
break;
case "while":
System.out.print("6 ");
break;
}
}else{
System.out.print("27 ");
}
}
public static void tokenize(String line){
StringBuffer consumed = new StringBuffer();
outerloop:
for (int i = 0; i < line.length(); i++) {
char ch = line.charAt(i);
if(isLetter(ch) && consumed.length() == 0){
consumed.append(line.charAt(i));
for (int j = i+1; j < line.length(); j++) {
ch = line.charAt(j);
if(isLetter(ch) || isNumber(ch)){
consumed.append(ch);
}else{
//call lexme to tokenize string
lexMe(consumed);
//consumed.setLength(0);
i = j;
}
}
}else if(isNumber(ch) && consumed.length() == 0){
consumed.append(line.charAt(i) );
for (int j = i+1; j < line.length(); j++) {
ch = line.charAt(j);
if(isNumber(ch) || line.charAt(j) == '.'){
consumed.append(ch);
}else{
System.out.print("28 ");
i = j;
}
}
}else if (isSymbol(ch)){
switch(ch){
case '+':
System.out.print("7 ");
break;
case '-':
System.out.print("8 ");
break;
case '*':
if(line.charAt(i-1) == '/'){
break outerloop;
}else{
System.out.println("9 ");
}
break;
case '/':
if(line.charAt(i+1) == '/')
break outerloop;
else if((ch = line.charAt(i+1)) == '*'){
consumed.append(ch);
for (int j = i; j < line.length(); j++) {
ch = line.charAt(j);
if(ch == '*'){
if(ch == '/'){
break outerloop;
}
}else{
consumed.append(ch);
}
}
}else{
System.out.println("10 ");
}
break;
case '<':
if(line.charAt(i+1) == '='){
System.out.print("12 ");
break;
}
System.out.print("11 ");
break;
case '>':
if(line.charAt(i+1) == '='){
System.out.print("14 ");
break;
}
System.out.print("13 ");
break;
case '!':
if(line.charAt(i+1) == '='){
System.out.print("16 ");
break;
}
break;
case '=':
System.out.print("15 ");
break;
case ';':
System.out.print("18 ");
break;
case ',':
System.out.print("19 ");
break;
case '.':
System.out.print("20 ");
break;
case '(':
System.out.print("21 ");
break;
case ')':
System.out.print("22 ");
break;
case '[':
System.out.print("23 ");
break;
case ']':
System.out.print("24 ");
break;
case '{':
System.out.print("25 ");
break;
case '}':
System.out.print("26 ");
break;
}
}
}
System.out.println("");
}
public static void main(String[] args) throws FileNotFoundException, IOException {
File file = new File("src\\testCode.txt");
String testCode;
try {
Scanner scanner = new Scanner(file);
while (scanner.hasNextLine()) {
String line = scanner.nextLine();
tokenize(line);
}
scanner.close();
}catch (FileNotFoundException e) {
}
}
}
公共类cmmLex{
公共静态布尔值（字符b）{
字符[]字母{'a'，'b'，'c'，'d'，'e'，'f'，'g'，'h'，'i'，'j'，'k'，'l'，'m'，'n'，'o'，'p'，'q'，'r'，'s'，'t'，'u'，'v'，'w'，'x'，'y'，'z'，'a'，'b'，'c'，'d'，
‘E’、‘F’、‘G’、‘H’、‘I’、‘J’、‘K’、‘L’、‘M’、‘N’、‘O’、‘P’、‘Q’、‘R’、‘S’、‘T’、‘U’、‘V’、‘W’、‘X’、‘Y’、‘Z’；
for（int i=0；i<字母长度；i++）{
if（b==字母[i]）
返回true；
}
返回false；
}
公共静态布尔isNumber（字符b）{
字符[]数={0'、'1'、'2'、'3'、'4'、'5'、'6'、'7'、'8'、'9'}；
for（int i=0；i consumed.setLength(0);
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.Scanner;
public class cmmLex {
public static boolean isLetter(char b){
char[] letters_ = {'a','b','c','d','e','f','g','h','i','j','k','l','m','n','o','p','q','r','s','t','u','v','w','x','y','z','A','B','C','D',
'E','F','G','H','I','J','K','L','M','N','O','P','Q','R','S','T','U','V','W','X','Y','Z','_'};
for (int i = 0; i < letters_.length; i++) {
if(b == letters_[i])
return true;
}
return false;
}
public static boolean isNumber(char b){
char[] numbers = {'0','1','2','3','4','5','6','7','8','9'};
for (int i = 0; i < numbers.length; i++) {
if(b == numbers[i])
return true;
}
return false;
}
public static boolean isKeyword(StringBuffer str){
String[] keywords = {"int", "double", "if", "while","return","void","else"};
for (int i = 0; i < keywords.length; i++) {
if (keywords[i].equals(str.toString()))
return true;
}
return false;
}
public static boolean isSymbol(char a){
char[] symbols = {'+','-','*','/','<','>','!','=',',','.','(',')','[',']','{','}'};
for (int i = 0; i < symbols.length; i++) {
if(a == symbols[i])
return true;
}
return false;
}
public static void lexMe(StringBuffer string)
{
if(isKeyword(string)){
switch(string.toString()){
case "double":
System.out.print("0 ");
break;
case "else":
System.out.print("1 ");
break;
case "if":
System.out.print("2 ");
break;
case "int":
System.out.print("3 ");
break;
case "return":
System.out.print("4 ");
break;
case "void":
System.out.print("5 ");
break;
case "while":
System.out.print("6 ");
break;
}
}else{
System.out.print("27 ");
}
}
public static void tokenize(String line){
StringBuffer consumed = new StringBuffer();
outerloop:
for (int i = 0; i < line.length(); i++) {
char ch = line.charAt(i);
if(isLetter(ch) && consumed.length() == 0){
consumed.append(line.charAt(i));
for (int j = i+1; j < line.length(); j++) {
ch = line.charAt(j);
if(isLetter(ch) || isNumber(ch)){
consumed.append(ch);
}else{
//call lexme to tokenize string
lexMe(consumed);
consumed.setLength(0);
i = j - 1;
break;
}
}
}else if(isNumber(ch) && consumed.length() == 0){
consumed.append(line.charAt(i) );
for (int j = i+1; j < line.length(); j++) {
ch = line.charAt(j);
if(isNumber(ch) || line.charAt(j) == '.'){
consumed.append(ch);
}else{
System.out.print("28 ");
consumed.setLength(0);
i = j - 1;
break;
}
}
}else if (isSymbol(ch)){
switch(ch){
case '+':
System.out.print("7 ");
break;
case '-':
System.out.print("8 ");
break;
case '*':
if(line.charAt(i-1) == '/'){
break outerloop;
}else{
System.out.println("9 ");
}
break;
case '/':
if(line.charAt(i+1) == '/')
break outerloop;
else if((ch = line.charAt(i+1)) == '*'){
consumed.append(ch);
for (int j = i; j < line.length(); j++) {
ch = line.charAt(j);
if(ch == '*'){
if(ch == '/'){
break outerloop;
}
}else{
consumed.append(ch);
}
}
}else{
System.out.println("10 ");
}
break;
case '<':
if(line.charAt(i+1) == '='){
System.out.print("12 ");
break;
}
System.out.print("11 ");
break;
case '>':
if(line.charAt(i+1) == '='){
System.out.print("14 ");
break;
}
System.out.print("13 ");
break;
case '!':
if(line.charAt(i+1) == '='){
System.out.print("16 ");
break;
}
break;
case '=':
System.out.print("15 ");
break;
case ';':
System.out.print("18 ");
break;
case ',':
System.out.print("19 ");
break;
case '.':
System.out.print("20 ");
break;
case '(':
System.out.print("21 ");
break;
case ')':
System.out.print("22 ");
break;
case '[':
System.out.print("23 ");
break;
case ']':
System.out.print("24 ");
break;
case '{':
System.out.print("25 ");
break;
case '}':
System.out.print("26 ");
break;
}
}
}
System.out.println("");
}
public static void main(String[] args) throws FileNotFoundException, IOException {
File file = new File("src\\testCode.txt");
String testCode;
try {
Scanner scanner = new Scanner(file);
while (scanner.hasNextLine()) {
String line = scanner.nextLine();
tokenize(line);
}
scanner.close();
} catch (FileNotFoundException e) {
}
}
}