Current File : //lib/python3/dist-packages/dnslib/lex.py
# -*- coding: utf-8 -*-
from __future__ import print_function
import collections,string
try:
from StringIO import StringIO
except ImportError:
from io import StringIO
class Lexer(object):
"""
Simple Lexer base class. Provides basic lexer framework and
helper functionality (read/peek/pushback etc)
Each state is implemented using a method (lexXXXX) which should
match a single token and return a (token,lexYYYY) tuple, with lexYYYY
representing the next state. If token is None this is not emitted
and if lexYYYY is None or the lexer reaches the end of the
input stream the lexer exits.
The 'parse' method returns a generator that will return tokens
(the class also acts as an iterator)
The default start state is 'lexStart'
Input can either be a string/bytes or file object.
The approach is based loosely on Rob Pike's Go lexer presentation
(using generators rather than channels).
>>> p = Lexer("a bcd efgh")
>>> p.read()
'a'
>>> p.read()
' '
>>> p.peek(3)
'bcd'
>>> p.read(5)
'bcd e'
>>> p.pushback('e')
>>> p.read(4)
'efgh'
"""
escape_chars = '\\'
escape = {'n':'\n','t':'\t','r':'\r'}
def __init__(self,f,debug=False):
if hasattr(f,'read'):
self.f = f
elif type(f) == str:
self.f = StringIO(f)
elif type(f) == bytes:
self.f = StringIO(f.decode())
else:
raise ValueError("Invalid input")
self.debug = debug
self.q = collections.deque()
self.state = self.lexStart
self.escaped = False
self.eof = False
def __iter__(self):
return self.parse()
def next_token(self):
if self.debug:
print("STATE",self.state)
(tok,self.state) = self.state()
return tok
def parse(self):
while self.state is not None and not self.eof:
tok = self.next_token()
if tok:
yield tok
def read(self,n=1):
s = ""
while self.q and n > 0:
s += self.q.popleft()
n -= 1
s += self.f.read(n)
if s == '':
self.eof = True
if self.debug:
print("Read: >%s<" % repr(s))
return s
def peek(self,n=1):
s = ""
i = 0
while len(self.q) > i and n > 0:
s += self.q[i]
i += 1
n -= 1
r = self.f.read(n)
if n > 0 and r == '':
self.eof = True
self.q.extend(r)
if self.debug:
print("Peek : >%s<" % repr(s + r))
return s + r
def pushback(self,s):
p = collections.deque(s)
p.extend(self.q)
self.q = p
def readescaped(self):
c = self.read(1)
if c in self.escape_chars:
self.escaped = True
n = self.peek(3)
if n.isdigit():
n = self.read(3)
if self.debug:
print("Escape: >%s<" % n)
return chr(int(n,8))
elif n[0] in 'x':
x = self.read(3)
if self.debug:
print("Escape: >%s<" % x)
return chr(int(x[1:],16))
else:
c = self.read(1)
if self.debug:
print("Escape: >%s<" % c)
return self.escape.get(c,c)
else:
self.escaped = False
return c
def lexStart(self):
return (None,None)
class WordLexer(Lexer):
"""
Example lexer which will split input stream into words (respecting
quotes)
To emit SPACE tokens: self.spacetok = ('SPACE',None)
To emit NL tokens: self.nltok = ('NL',None)
>>> l = WordLexer(r'abc "def\100\x3d\. ghi" jkl')
>>> list(l)
[('ATOM', 'abc'), ('ATOM', 'def@=. ghi'), ('ATOM', 'jkl')]
>>> l = WordLexer(r"1 '2 3 4' 5")
>>> list(l)
[('ATOM', '1'), ('ATOM', '2 3 4'), ('ATOM', '5')]
>>> l = WordLexer("abc# a comment")
>>> list(l)
[('ATOM', 'abc'), ('COMMENT', 'a comment')]
"""
wordchars = set(string.ascii_letters) | set(string.digits) | \
set(string.punctuation)
quotechars = set('"\'')
commentchars = set('#')
spacechars = set(' \t\x0b\x0c')
nlchars = set('\r\n')
spacetok = None
nltok = None
def lexStart(self):
return (None,self.lexSpace)
def lexSpace(self):
s = []
if self.spacetok:
tok = lambda n : (self.spacetok,n) if s else (None,n)
else:
tok = lambda n : (None,n)
while not self.eof:
c = self.peek()
if c in self.spacechars:
s.append(self.read())
elif c in self.nlchars:
return tok(self.lexNL)
elif c in self.commentchars:
return tok(self.lexComment)
elif c in self.quotechars:
return tok(self.lexQuote)
elif c in self.wordchars:
return tok(self.lexWord)
elif c:
raise ValueError("Invalid input [%d]: %s" % (
self.f.tell(),c))
return (None,None)
def lexNL(self):
while True:
c = self.read()
if c not in self.nlchars:
self.pushback(c)
return (self.nltok,self.lexSpace)
def lexComment(self):
s = []
tok = lambda n : (('COMMENT',''.join(s)),n) if s else (None,n)
start = False
_ = self.read()
while not self.eof:
c = self.read()
if c == '\n':
self.pushback(c)
return tok(self.lexNL)
elif start or c not in string.whitespace:
start = True
s.append(c)
return tok(None)
def lexWord(self):
s = []
tok = lambda n : (('ATOM',''.join(s)),n) if s else (None,n)
while not self.eof:
c = self.peek()
if c == '"':
return tok(self.lexQuote)
elif c in self.commentchars:
return tok(self.lexComment)
elif c.isspace():
return tok(self.lexSpace)
elif c in self.wordchars:
s.append(self.read())
elif c:
raise ValueError('Invalid input [%d]: %s' % (
self.f.tell(),c))
return tok(None)
def lexQuote(self):
s = []
tok = lambda n : (('ATOM',''.join(s)),n)
q = self.read(1)
while not self.eof:
c = self.readescaped()
if c == q and not self.escaped:
break
else:
s.append(c)
return tok(self.lexSpace)
class RandomLexer(Lexer):
"""
Test lexing from infinite stream.
Extract strings of letters/numbers from /dev/urandom
>>> import itertools,sys
>>> if sys.version[0] == '2':
... f = open("/dev/urandom")
... else:
... f = open("/dev/urandom",encoding="ascii",errors="replace")
>>> r = RandomLexer(f)
>>> i = iter(r)
>>> len(list(itertools.islice(i,10)))
10
"""
minalpha = 4
mindigits = 3
def lexStart(self):
return (None,self.lexRandom)
def lexRandom(self):
n = 0
c = self.peek(1)
while not self.eof:
if c.isalpha():
return (None,self.lexAlpha)
elif c.isdigit():
return (None,self.lexDigits)
else:
n += 1
_ = self.read(1)
c = self.peek(1)
return (None,None)
def lexDigits(self):
s = []
c = self.read(1)
while c.isdigit():
s.append(c)
c = self.read(1)
self.pushback(c)
if len(s) >= self.mindigits:
return (('NUMBER',"".join(s)),self.lexRandom)
else:
return (None,self.lexRandom)
def lexAlpha(self):
s = []
c = self.read(1)
while c.isalpha():
s.append(c)
c = self.read(1)
self.pushback(c)
if len(s) >= self.minalpha:
return (('STRING',"".join(s)),self.lexRandom)
else:
return (None,self.lexRandom)
if __name__ == '__main__':
import argparse,doctest,sys
p = argparse.ArgumentParser(description="Lex Tester")
p.add_argument("--lex","-l",action='store_true',default=False,
help="Lex input (stdin)")
p.add_argument("--nl",action='store_true',default=False,
help="Output NL tokens")
p.add_argument("--space",action='store_true',default=False,
help="Output Whitespace tokens")
p.add_argument("--wordchars",help="Wordchars")
p.add_argument("--quotechars",help="Quotechars")
p.add_argument("--commentchars",help="Commentchars")
p.add_argument("--spacechars",help="Spacechars")
p.add_argument("--nlchars",help="NLchars")
args = p.parse_args()
if args.lex:
l = WordLexer(sys.stdin)
if args.wordchars:
l.wordchars = set(args.wordchars)
if args.quotechars:
l.quotechars = set(args.quotechars)
if args.commentchars:
l.commentchars = set(args.commentchars)
if args.spacechars:
l.spacechars = set(args.spacechars)
if args.nlchars:
l.nlchars = set(args.nlchars)
if args.space:
l.spacetok = ('SPACE',)
if args.nl:
l.nltok = ('NL',)
for tok in l:
print(tok)
else:
try:
# Test if we have /dev/urandom
open("/dev/urandom")
sys.exit(0 if doctest.testmod().failed == 0 else 1)
except IOError:
# Don't run stream test
doctest.run_docstring_examples(Lexer, globals())
doctest.run_docstring_examples(WordLexer, globals())
Mini Shell