手工构造词法分析器

手工构造词法分析器词法分析器的实现方法手工编码词法分析器的生成器手工编码的方法相对复杂并且容易出错但是它是目前流行的实现方法。词法分析器生成器简单一些但是难以控制细节。这篇笔记主要记录了手工构造词法分析器。状态转移图先说有限自动机FA也叫又穷自动机是编译原理中描述词法规则、识别正则语言的一种抽象计算模型。状态转移图是有限自动机的图形化表示最常用于词法分析阶段用来描述如何识别各类单词符号token在词法分析器无非就是实现了一张状态转移图或者说把词法分析器用代码的形式写出来就得到了词法分析器。词法分析器示例下面用python实现了一个类C语言的词法分析器fromenumimportEnum,autofromtypingimportNamedTupleclassTokenType(Enum):KEYWORDauto()IDENTIFIERauto()INTEGERauto()FLOATauto()STRINGauto()OPERATORauto()DELIMITERauto()COMMENTauto()EOFauto()classToken(NamedTuple):type:TokenType value:strline:intcolumn:int# 关键字集合KEYWORDS{if,else,while,for,return,def,class,import,from,in,not,and,or,True,False,int,float,string,bool,void,let,var,fn,}# 运算符集合OPERATORS{,-,*,/,%,,,!,,,,,,||,!,,|,^,~,,,,-,*,/,,--,}# 分隔符集合DELIMITERS{(,),{,},[,],,,;,.,:}classLexerError(Exception):def__init__(self,message:str,line:int,column:int):self.lineline self.columncolumnsuper().__init__(f[行{line}, 列{column}]{message})classLexer:def__init__(self,source:str):self.sourcesource self.pos0self.line1self.column1def_peek(self)-str:ifself.poslen(self.source):return\0returnself.source[self.pos]def_advance(self)-str:chself.source[self.pos]self.pos1ifch\n:self.line1self.column1else:self.column1returnchdef_skip_whitespace(self):whileself.poslen(self.source)andself.source[self.pos]in \t\r\n:self._advance()def_read_string(self,quote:str)-Token:start_line,start_colself.line,self.column self._advance()# skip opening quotebuf[]whileself.poslen(self.source):chself.source[self.pos]ifch\\:self._advance()ifself.poslen(self.source):escself._advance()escape_map{n:\n,t:\t,r:\r,\\:\\,0:\0}buf.append(escape_map.get(esc,esc))continueifchquote:self._advance()returnToken(TokenType.STRING,.join(buf),start_line,start_col)buf.append(ch)self._advance()raiseLexerError(未终止的字符串字面量,start_line,start_col)def_read_number(self)-Token:start_line,start_colself.line,self.column buf[]whileself.poslen(self.source)andself.source[self.pos].isdigit():buf.append(self._advance())ifself.poslen(self.source)andself.source[self.pos].:next_posself.pos1ifnext_poslen(self.source)andself.source[next_pos].isdigit():buf.append(self._advance())# .whileself.poslen(self.source)andself.source[self.pos].isdigit():buf.append(self._advance())returnToken(TokenType.FLOAT,.join(buf),start_line,start_col)returnToken(TokenType.INTEGER,.join(buf),start_line,start_col)def_read_identifier_or_keyword(self)-Token:start_line,start_colself.line,self.column buf[self._advance()]whileself.poslen(self.source)and(self.source[self.pos].isalnum()orself.source[self.pos]_):buf.append(self._advance())word.join(buf)ifwordinKEYWORDS:returnToken(TokenType.KEYWORD,word,start_line,start_col)returnToken(TokenType.IDENTIFIER,word,start_line,start_col)def_read_comment(self)-Token:start_line,start_colself.line,self.column buf[self._advance(),self._advance()]# //whileself.poslen(self.source)andself.source[self.pos]!\n:buf.append(self._advance())returnToken(TokenType.COMMENT,.join(buf),start_line,start_col)def_read_operator_or_delimiter(self)-Token:start_line,start_colself.line,self.column chself.source[self.pos]ifchinDELIMITERS:self._advance()returnToken(TokenType.DELIMITER,ch,start_line,start_col)twoself.source[self.pos:self.pos2]iftwoinOPERATORS:self._advance()self._advance()returnToken(TokenType.OPERATOR,two,start_line,start_col)ifchinOPERATORS:self._advance()returnToken(TokenType.OPERATOR,ch,start_line,start_col)raiseLexerError(f无法识别的字符: {ch},self.line,self.column)deftokenize(self)-list[Token]:tokens[]whileself.poslen(self.source):self._skip_whitespace()ifself.poslen(self.source):breakchself.source[self.pos]ifch/andself.pos1len(self.source)andself.source[self.pos1]/:tokens.append(self._read_comment())elifch/andself.pos1len(self.source)andself.source[self.pos1]*:self._advance()self._advance()start_line,start_colself.line,self.column buf[/*]whileself.poslen(self.source):ifself.source[self.pos]*andself.pos1len(self.source)andself.source[self.pos1]/:buf.append(*/)self._advance()self._advance()breakbuf.append(self.source[self.pos])self._advance()else:raiseLexerError(未终止的块注释,start_line,start_col)tokens.append(Token(TokenType.COMMENT,.join(buf),start_line,start_col))elifchin(,):tokens.append(self._read_string(ch))elifch.isdigit():tokens.append(self._read_number())elifch.isalpha()orch_:tokens.append(self._read_identifier_or_keyword())else:tokens.append(self._read_operator_or_delimiter())tokens.append(Token(TokenType.EOF,,self.line,self.column))returntokensdefformat_token(token:Token)-str:returnf({token.type.name:10s}|{token.line:3d}:{token.column:3d}|{token.value!r})if__name____main__:importsysiflen(sys.argv)2:print(用法: python lexer.py 输入文件)sys.exit(1)withopen(sys.argv[1],r,encodingutf-8)asf:sourcef.read()lexerLexer(source)try:tokenslexer.tokenize()exceptLexerErrorase:print(f词法错误:{e},filesys.stderr)sys.exit(1)fortokintokens:iftok.typeTokenType.EOF:breakprint(format_token(tok))