| Current File : /home/mmdealscpanel/yummmdeals.com/pgen2.tar |
__init__.py 0000644 00000000217 15046736237 0006672 0 ustar 00 # Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved.
# Licensed to PSF under a Contributor Agreement.
"""The pgen2 package."""
parse.py 0000644 00000017733 15046736237 0006260 0 ustar 00 # Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved.
# Licensed to PSF under a Contributor Agreement.
"""Parser engine for the grammar tables generated by pgen.
The grammar table must be loaded first.
See Parser/parser.c in the Python distribution for additional info on
how this parsing engine works.
"""
# Local imports
from . import token
class ParseError(Exception):
"""Exception to signal the parser is stuck."""
def __init__(self, msg, type, value, context):
Exception.__init__(self, "%s: type=%r, value=%r, context=%r" %
(msg, type, value, context))
self.msg = msg
self.type = type
self.value = value
self.context = context
def __reduce__(self):
return type(self), (self.msg, self.type, self.value, self.context)
class Parser(object):
"""Parser engine.
The proper usage sequence is:
p = Parser(grammar, [converter]) # create instance
p.setup([start]) # prepare for parsing
<for each input token>:
if p.addtoken(...): # parse a token; may raise ParseError
break
root = p.rootnode # root of abstract syntax tree
A Parser instance may be reused by calling setup() repeatedly.
A Parser instance contains state pertaining to the current token
sequence, and should not be used concurrently by different threads
to parse separate token sequences.
See driver.py for how to get input tokens by tokenizing a file or
string.
Parsing is complete when addtoken() returns True; the root of the
abstract syntax tree can then be retrieved from the rootnode
instance variable. When a syntax error occurs, addtoken() raises
the ParseError exception. There is no error recovery; the parser
cannot be used after a syntax error was reported (but it can be
reinitialized by calling setup()).
"""
def __init__(self, grammar, convert=None):
"""Constructor.
The grammar argument is a grammar.Grammar instance; see the
grammar module for more information.
The parser is not ready yet for parsing; you must call the
setup() method to get it started.
The optional convert argument is a function mapping concrete
syntax tree nodes to abstract syntax tree nodes. If not
given, no conversion is done and the syntax tree produced is
the concrete syntax tree. If given, it must be a function of
two arguments, the first being the grammar (a grammar.Grammar
instance), and the second being the concrete syntax tree node
to be converted. The syntax tree is converted from the bottom
up.
A concrete syntax tree node is a (type, value, context, nodes)
tuple, where type is the node type (a token or symbol number),
value is None for symbols and a string for tokens, context is
None or an opaque value used for error reporting (typically a
(lineno, offset) pair), and nodes is a list of children for
symbols, and None for tokens.
An abstract syntax tree node may be anything; this is entirely
up to the converter function.
"""
self.grammar = grammar
self.convert = convert or (lambda grammar, node: node)
def setup(self, start=None):
"""Prepare for parsing.
This *must* be called before starting to parse.
The optional argument is an alternative start symbol; it
defaults to the grammar's start symbol.
You can use a Parser instance to parse any number of programs;
each time you call setup() the parser is reset to an initial
state determined by the (implicit or explicit) start symbol.
"""
if start is None:
start = self.grammar.start
# Each stack entry is a tuple: (dfa, state, node).
# A node is a tuple: (type, value, context, children),
# where children is a list of nodes or None, and context may be None.
newnode = (start, None, None, [])
stackentry = (self.grammar.dfas[start], 0, newnode)
self.stack = [stackentry]
self.rootnode = None
self.used_names = set() # Aliased to self.rootnode.used_names in pop()
def addtoken(self, type, value, context):
"""Add a token; return True iff this is the end of the program."""
# Map from token to label
ilabel = self.classify(type, value, context)
# Loop until the token is shifted; may raise exceptions
while True:
dfa, state, node = self.stack[-1]
states, first = dfa
arcs = states[state]
# Look for a state with this label
for i, newstate in arcs:
t, v = self.grammar.labels[i]
if ilabel == i:
# Look it up in the list of labels
assert t < 256
# Shift a token; we're done with it
self.shift(type, value, newstate, context)
# Pop while we are in an accept-only state
state = newstate
while states[state] == [(0, state)]:
self.pop()
if not self.stack:
# Done parsing!
return True
dfa, state, node = self.stack[-1]
states, first = dfa
# Done with this token
return False
elif t >= 256:
# See if it's a symbol and if we're in its first set
itsdfa = self.grammar.dfas[t]
itsstates, itsfirst = itsdfa
if ilabel in itsfirst:
# Push a symbol
self.push(t, self.grammar.dfas[t], newstate, context)
break # To continue the outer while loop
else:
if (0, state) in arcs:
# An accepting state, pop it and try something else
self.pop()
if not self.stack:
# Done parsing, but another token is input
raise ParseError("too much input",
type, value, context)
else:
# No success finding a transition
raise ParseError("bad input", type, value, context)
def classify(self, type, value, context):
"""Turn a token into a label. (Internal)"""
if type == token.NAME:
# Keep a listing of all used names
self.used_names.add(value)
# Check for reserved words
ilabel = self.grammar.keywords.get(value)
if ilabel is not None:
return ilabel
ilabel = self.grammar.tokens.get(type)
if ilabel is None:
raise ParseError("bad token", type, value, context)
return ilabel
def shift(self, type, value, newstate, context):
"""Shift a token. (Internal)"""
dfa, state, node = self.stack[-1]
newnode = (type, value, context, None)
newnode = self.convert(self.grammar, newnode)
if newnode is not None:
node[-1].append(newnode)
self.stack[-1] = (dfa, newstate, node)
def push(self, type, newdfa, newstate, context):
"""Push a nonterminal. (Internal)"""
dfa, state, node = self.stack[-1]
newnode = (type, None, context, [])
self.stack[-1] = (dfa, newstate, node)
self.stack.append((newdfa, 0, newnode))
def pop(self):
"""Pop a nonterminal. (Internal)"""
popdfa, popstate, popnode = self.stack.pop()
newnode = self.convert(self.grammar, popnode)
if newnode is not None:
if self.stack:
dfa, state, node = self.stack[-1]
node[-1].append(newnode)
else:
self.rootnode = newnode
self.rootnode.used_names = self.used_names
conv.py 0000644 00000022652 15046736237 0006107 0 ustar 00 # Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved.
# Licensed to PSF under a Contributor Agreement.
"""Convert graminit.[ch] spit out by pgen to Python code.
Pgen is the Python parser generator. It is useful to quickly create a
parser from a grammar file in Python's grammar notation. But I don't
want my parsers to be written in C (yet), so I'm translating the
parsing tables to Python data structures and writing a Python parse
engine.
Note that the token numbers are constants determined by the standard
Python tokenizer. The standard token module defines these numbers and
their names (the names are not used much). The token numbers are
hardcoded into the Python tokenizer and into pgen. A Python
implementation of the Python tokenizer is also available, in the
standard tokenize module.
On the other hand, symbol numbers (representing the grammar's
non-terminals) are assigned by pgen based on the actual grammar
input.
Note: this module is pretty much obsolete; the pgen module generates
equivalent grammar tables directly from the Grammar.txt input file
without having to invoke the Python pgen C program.
"""
# Python imports
import re
# Local imports
from pgen2 import grammar, token
class Converter(grammar.Grammar):
"""Grammar subclass that reads classic pgen output files.
The run() method reads the tables as produced by the pgen parser
generator, typically contained in two C files, graminit.h and
graminit.c. The other methods are for internal use only.
See the base class for more documentation.
"""
def run(self, graminit_h, graminit_c):
"""Load the grammar tables from the text files written by pgen."""
self.parse_graminit_h(graminit_h)
self.parse_graminit_c(graminit_c)
self.finish_off()
def parse_graminit_h(self, filename):
"""Parse the .h file written by pgen. (Internal)
This file is a sequence of #define statements defining the
nonterminals of the grammar as numbers. We build two tables
mapping the numbers to names and back.
"""
try:
f = open(filename)
except OSError as err:
print("Can't open %s: %s" % (filename, err))
return False
self.symbol2number = {}
self.number2symbol = {}
lineno = 0
for line in f:
lineno += 1
mo = re.match(r"^#define\s+(\w+)\s+(\d+)$", line)
if not mo and line.strip():
print("%s(%s): can't parse %s" % (filename, lineno,
line.strip()))
else:
symbol, number = mo.groups()
number = int(number)
assert symbol not in self.symbol2number
assert number not in self.number2symbol
self.symbol2number[symbol] = number
self.number2symbol[number] = symbol
return True
def parse_graminit_c(self, filename):
"""Parse the .c file written by pgen. (Internal)
The file looks as follows. The first two lines are always this:
#include "pgenheaders.h"
#include "grammar.h"
After that come four blocks:
1) one or more state definitions
2) a table defining dfas
3) a table defining labels
4) a struct defining the grammar
A state definition has the following form:
- one or more arc arrays, each of the form:
static arc arcs_<n>_<m>[<k>] = {
{<i>, <j>},
...
};
- followed by a state array, of the form:
static state states_<s>[<t>] = {
{<k>, arcs_<n>_<m>},
...
};
"""
try:
f = open(filename)
except OSError as err:
print("Can't open %s: %s" % (filename, err))
return False
# The code below essentially uses f's iterator-ness!
lineno = 0
# Expect the two #include lines
lineno, line = lineno+1, next(f)
assert line == '#include "pgenheaders.h"\n', (lineno, line)
lineno, line = lineno+1, next(f)
assert line == '#include "grammar.h"\n', (lineno, line)
# Parse the state definitions
lineno, line = lineno+1, next(f)
allarcs = {}
states = []
while line.startswith("static arc "):
while line.startswith("static arc "):
mo = re.match(r"static arc arcs_(\d+)_(\d+)\[(\d+)\] = {$",
line)
assert mo, (lineno, line)
n, m, k = list(map(int, mo.groups()))
arcs = []
for _ in range(k):
lineno, line = lineno+1, next(f)
mo = re.match(r"\s+{(\d+), (\d+)},$", line)
assert mo, (lineno, line)
i, j = list(map(int, mo.groups()))
arcs.append((i, j))
lineno, line = lineno+1, next(f)
assert line == "};\n", (lineno, line)
allarcs[(n, m)] = arcs
lineno, line = lineno+1, next(f)
mo = re.match(r"static state states_(\d+)\[(\d+)\] = {$", line)
assert mo, (lineno, line)
s, t = list(map(int, mo.groups()))
assert s == len(states), (lineno, line)
state = []
for _ in range(t):
lineno, line = lineno+1, next(f)
mo = re.match(r"\s+{(\d+), arcs_(\d+)_(\d+)},$", line)
assert mo, (lineno, line)
k, n, m = list(map(int, mo.groups()))
arcs = allarcs[n, m]
assert k == len(arcs), (lineno, line)
state.append(arcs)
states.append(state)
lineno, line = lineno+1, next(f)
assert line == "};\n", (lineno, line)
lineno, line = lineno+1, next(f)
self.states = states
# Parse the dfas
dfas = {}
mo = re.match(r"static dfa dfas\[(\d+)\] = {$", line)
assert mo, (lineno, line)
ndfas = int(mo.group(1))
for i in range(ndfas):
lineno, line = lineno+1, next(f)
mo = re.match(r'\s+{(\d+), "(\w+)", (\d+), (\d+), states_(\d+),$',
line)
assert mo, (lineno, line)
symbol = mo.group(2)
number, x, y, z = list(map(int, mo.group(1, 3, 4, 5)))
assert self.symbol2number[symbol] == number, (lineno, line)
assert self.number2symbol[number] == symbol, (lineno, line)
assert x == 0, (lineno, line)
state = states[z]
assert y == len(state), (lineno, line)
lineno, line = lineno+1, next(f)
mo = re.match(r'\s+("(?:\\\d\d\d)*")},$', line)
assert mo, (lineno, line)
first = {}
rawbitset = eval(mo.group(1))
for i, c in enumerate(rawbitset):
byte = ord(c)
for j in range(8):
if byte & (1<<j):
first[i*8 + j] = 1
dfas[number] = (state, first)
lineno, line = lineno+1, next(f)
assert line == "};\n", (lineno, line)
self.dfas = dfas
# Parse the labels
labels = []
lineno, line = lineno+1, next(f)
mo = re.match(r"static label labels\[(\d+)\] = {$", line)
assert mo, (lineno, line)
nlabels = int(mo.group(1))
for i in range(nlabels):
lineno, line = lineno+1, next(f)
mo = re.match(r'\s+{(\d+), (0|"\w+")},$', line)
assert mo, (lineno, line)
x, y = mo.groups()
x = int(x)
if y == "0":
y = None
else:
y = eval(y)
labels.append((x, y))
lineno, line = lineno+1, next(f)
assert line == "};\n", (lineno, line)
self.labels = labels
# Parse the grammar struct
lineno, line = lineno+1, next(f)
assert line == "grammar _PyParser_Grammar = {\n", (lineno, line)
lineno, line = lineno+1, next(f)
mo = re.match(r"\s+(\d+),$", line)
assert mo, (lineno, line)
ndfas = int(mo.group(1))
assert ndfas == len(self.dfas)
lineno, line = lineno+1, next(f)
assert line == "\tdfas,\n", (lineno, line)
lineno, line = lineno+1, next(f)
mo = re.match(r"\s+{(\d+), labels},$", line)
assert mo, (lineno, line)
nlabels = int(mo.group(1))
assert nlabels == len(self.labels), (lineno, line)
lineno, line = lineno+1, next(f)
mo = re.match(r"\s+(\d+)$", line)
assert mo, (lineno, line)
start = int(mo.group(1))
assert start in self.number2symbol, (lineno, line)
self.start = start
lineno, line = lineno+1, next(f)
assert line == "};\n", (lineno, line)
try:
lineno, line = lineno+1, next(f)
except StopIteration:
pass
else:
assert 0, (lineno, line)
def finish_off(self):
"""Create additional useful structures. (Internal)."""
self.keywords = {} # map from keyword strings to arc labels
self.tokens = {} # map from numeric token values to arc labels
for ilabel, (type, value) in enumerate(self.labels):
if type == token.NAME and value is not None:
self.keywords[value] = ilabel
elif value is None:
self.tokens[type] = ilabel
__pycache__/token.cpython-36.opt-1.pyc 0000644 00000003474 15046736237 0013506 0 ustar 00 3
\ � @ sP d Z dZdZdZdZdZdZdZdZd Z d
Z
d Z d Z d
Z
dZdZdZdZdZdZdZdZdZdZdZdZdZdZd Z d Z d Z d Z d Z d!Z!d"Z"d#Z#d$Z$d%Z%d&Z&d'Z'd(Z(d)Z)d*Z*d+Z+d,Z,d-Z-d.Z.d/Z/d0Z0d1Z1d2Z2d3Z3d4Z4d5Z5d6Z6d7Z7d8Z8d9Z9d:Z:d;Z;d<Z<d=Z=i Z>x6e?e@� jA� �D ]$\ZBZCeDeC�eDd�k�r eBe>eC< �q W d>d?� ZEd@dA� ZFdBdC� ZGdDS )Ez!Token constants (from "token.h").� � � � � � � � � � �
� � �
� � � � � � � � � � � � � � � � � � � �! �" �# �$ �% �& �' �( �) �* �+ �, �- �. �/ �0 �1 �2 �3 �4 �5 �6 �7 �8 �9 �: �; � c C s | t k S )N)� NT_OFFSET)�x� r@ �+/usr/lib64/python3.6/lib2to3/pgen2/token.py�
ISTERMINALN s rB c C s | t kS )N)r> )r? r@ r@ rA �
ISNONTERMINALQ s rC c C s | t kS )N)� ENDMARKER)r? r@ r@ rA �ISEOFT s rE N)H�__doc__rD �NAME�NUMBER�STRING�NEWLINE�INDENT�DEDENT�LPAR�RPAR�LSQB�RSQB�COLON�COMMA�SEMI�PLUS�MINUS�STAR�SLASH�VBAR�AMPER�LESS�GREATER�EQUAL�DOT�PERCENTZ BACKQUOTE�LBRACE�RBRACE�EQEQUAL�NOTEQUAL� LESSEQUAL� GREATEREQUAL�TILDE�
CIRCUMFLEX� LEFTSHIFT�
RIGHTSHIFT�
DOUBLESTAR� PLUSEQUAL�MINEQUAL� STAREQUAL�
SLASHEQUAL� PERCENTEQUAL�
AMPEREQUAL� VBAREQUAL�CIRCUMFLEXEQUAL�LEFTSHIFTEQUAL�RIGHTSHIFTEQUAL�DOUBLESTAREQUAL� DOUBLESLASH�DOUBLESLASHEQUAL�AT�ATEQUAL�OP�COMMENT�NL�RARROW�AWAIT�ASYNC�
ERRORTOKEN�N_TOKENSr> �tok_name�list�globals�items�_nameZ_value�typerB rC rE r@ r@ r@ rA �<module> s� __pycache__/pgen.cpython-36.pyc 0000644 00000023071 15046736237 0012353 0 ustar 00 3
\�5 � @ sd d dl mZmZmZ G dd� dej�ZG dd� de�ZG dd� de�ZG dd � d e�Z dd d �Z
d
S )� )�grammar�token�tokenizec @ s e Zd ZdS )� PgenGrammarN)�__name__�
__module__� __qualname__� r r �*/usr/lib64/python3.6/lib2to3/pgen2/pgen.pyr s r c @ s� e Zd Zd&dd�Zdd� Zdd� Zdd � Zd
d � Zd d
� Zdd� Z dd� Z
dd� Z dd� Z dd� Z
dd� Zdd� Zd d � Zd d � Zd'd d!�Zd"d#� Zd$d%� ZdS )(�ParserGeneratorNc C sl d }|d krt |�}|j}|| _|| _tj|j�| _| j� | j � \| _
| _ |d k rZ|� i | _ | j
� d S )N)�open�close�filename�streamr �generate_tokens�readline� generator�gettoken�parse�dfas� startsymbol�first� addfirstsets)�selfr r Z close_streamr r r
�__init__ s zParserGenerator.__init__c C s* t � }t| jj� �}|j� |j| j� |jd| j� x.|D ]&}dt|j � }||j |< ||j
|< q<W x�|D ]�}| j| }g }xl|D ]d}g }x6t |j j
� �D ]$\} }
|j| j|| �|j|
�f� q�W |jr�|jd|j|�f� |j|� q�W |jj|� || j||�f|j|j | < qlW |j | j |_|S )N� � )r �listr �keys�sort�remover �insert�len�
symbol2numberZ
number2symbol�sorted�arcs�items�append�
make_label�index�isfinal�states�
make_first�start) r �c�names�name�i�dfar+ �stater% �label�nextr r r
� make_grammar s.
z ParserGenerator.make_grammarc C s8 | j | }i }x$t|�D ]}| j||�}d||< qW |S )Nr )r r$ r( )r r. r0 Zrawfirstr r4 �ilabelr r r
r, 4 s
zParserGenerator.make_firstc C sb t |j�}|d j� r�||jkrZ||jkr4|j| S |jj|j| d f� ||j|< |S nbtt|d �}t|t �sxt
|��|tj ks�t
|��||j kr�|j | S |jj|d f� ||j |< |S n�|d dks�t
|��t
|�}|d j� �r ||jk� r�|j| S |jjtj|f� ||j|< |S n>tj| }||j k�r@|j | S |jj|d f� ||j |< |S d S )Nr �"�')r8 r9 )r" Zlabels�isalphar# Z symbol2labelr' �getattrr �
isinstance�int�AssertionError�tok_name�tokens�eval�keywords�NAMEr Zopmap)r r. r4 r7 Zitoken�valuer r r
r( = s<
zParserGenerator.make_labelc C s< t | jj� �}|j� x |D ]}|| jkr | j|� q W d S )N)r r r r r � calcfirst)r r/ r0 r r r
r k s
z ParserGenerator.addfirstsetsc C s
| j | }d | j|< |d }i }i }x�|jj� D ]x\}}|| j kr�|| jkrl| j| }|d kr�td| ��n| j|� | j| }|j|� |||< q0d||< |di||< q0W i } xJ|j� D ]>\}}
x4|
D ],} | | kr�td|| || | f ��|| | < q�W q�W || j|< d S )Nr zrecursion for rule %rr zArule %s is ambiguous; %s is in the first sets of %s as well as %s)r r r% r& �
ValueErrorrE �update) r r0 r2 r3 ZtotalsetZ overlapcheckr4 r5 �fsetZinverseZitsfirstZsymbolr r r
rE s s2
zParserGenerator.calcfirstc C s� i }d }x�| j tjkr�x| j tjkr.| j� qW | jtj�}| jtjd� | j� \}}| jtj� | j ||�}t
|�}| j |� t
|�}|||< |d kr
|}q
W ||fS )N�:) �typer � ENDMARKER�NEWLINEr �expectrC �OP� parse_rhs�make_dfar" � simplify_dfa) r r r r0 �a�zr2 ZoldlenZnewlenr r r
r � s"
zParserGenerator.parsec s� t |t�st�t |t�s t�� fdd�}� fdd�� t||�|�g}x�|D ]�}i }x<|jD ]2}x,|jD ]"\}} |d k rf� | |j|i �� qfW qZW xRt|j� �D ]B\}}
x,|D ]} | j|
kr�P q�W t|
|�} |j | � |j
| |� q�W qJW |S )Nc s i }� | |� |S )Nr )r3 �base)�
addclosurer r
�closure� s
z)ParserGenerator.make_dfa.<locals>.closurec sL t | t�st�| |krd S d|| < x$| jD ]\}}|d kr*� ||� q*W d S )Nr )r< �NFAStater> r% )r3 rT r4 r5 )rU r r
rU � s z,ParserGenerator.make_dfa.<locals>.addclosure) r< rW r> �DFAState�nfasetr% �
setdefaultr$ r& r' �addarc) r r- �finishrV r+ r3 r% Znfastater4 r5 rY �str )rU r
rP � s&
zParserGenerator.make_dfac
C s� t d|� |g}x�t|�D ]�\}}t d|||kr4dp6d� x^|jD ]T\}}||kr^|j|�} nt|�} |j|� |d kr�t d| � qBt d|| f � qBW qW d S )NzDump of NFA forz Statez(final)� z -> %dz %s -> %d)�print� enumerater% r) r" r' )
r r0 r- r\ Ztodor1 r3 r4 r5 �jr r r
�dump_nfa� s
zParserGenerator.dump_nfac C sl t d|� x\t|�D ]P\}}t d||jr,dp.d� x0t|jj� �D ] \}}t d||j|�f � qBW qW d S )NzDump of DFA forz Statez(final)r^ z %s -> %d)r_ r` r* r$ r% r& r) )r r0 r2 r1 r3 r4 r5 r r r
�dump_dfa� s
zParserGenerator.dump_dfac C s~ d}xt|rxd}xft |�D ]Z\}}xPt|d t|��D ]:}|| }||kr4||= x|D ]}|j||� qTW d}P q4W qW qW d S )NTFr )r` �ranger" �
unifystate)r r2 Zchangesr1 Zstate_ira Zstate_jr3 r r r
rQ � s
z ParserGenerator.simplify_dfac C s� | j � \}}| jdkr ||fS t� }t� }|j|� |j|� x6| jdkrt| j� | j � \}}|j|� |j|� q@W ||fS d S )N�|)� parse_altrD rW r[ r )r rR rS ZaaZzzr r r
rO � s
zParserGenerator.parse_rhsc C sP | j � \}}x:| jdks*| jtjtjfkrF| j � \}}|j|� |}qW ||fS )N�(�[)rh ri )�
parse_itemrD rJ r rC �STRINGr[ )r rR �br. �dr r r
rg
s
zParserGenerator.parse_altc C s� | j dkr>| j� | j� \}}| jtjd� |j|� ||fS | j� \}}| j }|dkr`||fS | j� |j|� |dkr�||fS ||fS d S )Nri �]�+�*)ro rp )rD r rO rM r rN r[ �
parse_atom)r rR rS rD r r r
rj s
zParserGenerator.parse_itemc C s� | j dkr4| j� | j� \}}| jtjd� ||fS | jtjtjfkrpt � }t � }|j
|| j � | j� ||fS | j d| j| j � d S )Nrh �)z+expected (...) or NAME or STRING, got %s/%s) rD r rO rM r rN rJ rC rk rW r[ � raise_error)r rR rS r r r
rq ( s
zParserGenerator.parse_atomc C sD | j |ks |d k r2| j|kr2| jd||| j | j� | j}| j� |S )Nzexpected %s/%s, got %s/%s)rJ rD rs r )r rJ rD r r r
rM 9 s zParserGenerator.expectc C sJ t | j�}x"|d tjtjfkr,t | j�}q W |\| _| _| _| _| _ d S )Nr )
r5 r r �COMMENT�NLrJ rD Zbegin�end�line)r �tupr r r
r A s
zParserGenerator.gettokenc G s^ |r8y || }W n& dj |gttt|�� �}Y nX t|| j| jd | jd | jf��d S )N� r r )�joinr �map�str� SyntaxErrorr rv rw )r �msg�argsr r r
rs H s zParserGenerator.raise_error)N)N)r r r r r6 r, r( r rE r rP rb rc rQ rO rg rj rq rM r rs r r r r
r
s$
. $
r c @ s e Zd Zdd� Zddd�ZdS )rW c C s
g | _ d S )N)r% )r r r r
r S s zNFAState.__init__Nc C s8 |d kst |t�st�t |t�s$t�| jj||f� d S )N)r< r| r> rW r% r' )r r5 r4 r r r
r[ V s zNFAState.addarc)N)r r r r r[ r r r r
rW Q s rW c @ s0 e Zd Zdd� Zdd� Zdd� Zdd� Zd Zd S )
rX c C sL t |t�st�t tt|��t�s$t�t |t�s2t�|| _||k| _i | _d S )N) r< �dictr> r5 �iterrW rY r* r% )r rY �finalr r r
r ] s
zDFAState.__init__c C s8 t |t�st�|| jks t�t |t�s*t�|| j|< d S )N)r< r| r> r% rX )r r5 r4 r r r
r[ e s zDFAState.addarcc C s. x(| j j� D ]\}}||kr || j |< q W d S )N)r% r& )r �old�newr4 r5 r r r
re k s zDFAState.unifystatec C sf t |t�st�| j|jkr dS t| j�t|j�kr6dS x*| jj� D ] \}}||jj|�k rBdS qBW dS )NFT)r< rX r> r* r" r% r& �get)r �otherr4 r5 r r r
�__eq__p s zDFAState.__eq__N)r r r r r[ re r� �__hash__r r r r
rX [ s
rX � Grammar.txtc C s t | �}|j� S )N)r r6 )r �pr r r
�generate_grammar� s r� N)r� ) r^ r r r ZGrammarr �objectr rW rX r� r r r r
�<module> s I
% __pycache__/pgen.cpython-36.opt-1.pyc 0000644 00000022165 15046736237 0013315 0 ustar 00 3
\�5 � @ sd d dl mZmZmZ G dd� dej�ZG dd� de�ZG dd� de�ZG dd � d e�Z dd d �Z
d
S )� )�grammar�token�tokenizec @ s e Zd ZdS )� PgenGrammarN)�__name__�
__module__� __qualname__� r r �*/usr/lib64/python3.6/lib2to3/pgen2/pgen.pyr s r c @ s� e Zd Zd&dd�Zdd� Zdd� Zdd � Zd
d � Zd d
� Zdd� Z dd� Z
dd� Z dd� Z dd� Z
dd� Zdd� Zd d � Zd d � Zd'd d!�Zd"d#� Zd$d%� ZdS )(�ParserGeneratorNc C sl d }|d krt |�}|j}|| _|| _tj|j�| _| j� | j � \| _
| _ |d k rZ|� i | _ | j
� d S )N)�open�close�filename�streamr �generate_tokens�readline� generator�gettoken�parse�dfas� startsymbol�first� addfirstsets)�selfr r Z close_streamr r r
�__init__ s zParserGenerator.__init__c C s* t � }t| jj� �}|j� |j| j� |jd| j� x.|D ]&}dt|j � }||j |< ||j
|< q<W x�|D ]�}| j| }g }xl|D ]d}g }x6t |j j
� �D ]$\} }
|j| j|| �|j|
�f� q�W |jr�|jd|j|�f� |j|� q�W |jj|� || j||�f|j|j | < qlW |j | j |_|S )N� � )r �listr �keys�sort�remover �insert�len�
symbol2numberZ
number2symbol�sorted�arcs�items�append�
make_label�index�isfinal�states�
make_first�start) r �c�names�name�i�dfar+ �stater% �label�nextr r r
� make_grammar s.
z ParserGenerator.make_grammarc C s8 | j | }i }x$t|�D ]}| j||�}d||< qW |S )Nr )r r$ r( )r r. r0 Zrawfirstr r4 �ilabelr r r
r, 4 s
zParserGenerator.make_firstc C s& t |j�}|d j� r�||jkrZ||jkr4|j| S |jj|j| d f� ||j|< |S n>tt|d �}||jkrz|j| S |jj|d f� ||j|< |S n�t |�}|d j� r�||j
kr�|j
| S |jjtj |f� ||j
|< |S n>t j
| }||jk�r|j| S |jj|d f� ||j|< |S d S )Nr )r" Zlabels�isalphar# Z symbol2labelr' �getattrr �tokens�eval�keywords�NAMEr Zopmap)r r. r4 r7 Zitoken�valuer r r
r( = s6
zParserGenerator.make_labelc C s< t | jj� �}|j� x |D ]}|| jkr | j|� q W d S )N)r r r r r � calcfirst)r r/ r0 r r r
r k s
z ParserGenerator.addfirstsetsc C s
| j | }d | j|< |d }i }i }x�|jj� D ]x\}}|| j kr�|| jkrl| j| }|d kr�td| ��n| j|� | j| }|j|� |||< q0d||< |di||< q0W i } xJ|j� D ]>\}}
x4|
D ],} | | kr�td|| || | f ��|| | < q�W q�W || j|< d S )Nr zrecursion for rule %rr zArule %s is ambiguous; %s is in the first sets of %s as well as %s)r r r% r& �
ValueErrorr? �update) r r0 r2 r3 ZtotalsetZ overlapcheckr4 r5 �fsetZinverseZitsfirstZsymbolr r r
r? s s2
zParserGenerator.calcfirstc C s� i }d }x�| j tjkr�x| j tjkr.| j� qW | jtj�}| jtjd� | j� \}}| jtj� | j ||�}t
|�}| j |� t
|�}|||< |d kr
|}q
W ||fS )N�:) �typer � ENDMARKER�NEWLINEr �expectr= �OP� parse_rhs�make_dfar" � simplify_dfa) r r r r0 �a�zr2 ZoldlenZnewlenr r r
r � s"
zParserGenerator.parsec s� � fdd�}� fdd�� t ||�|�g}x�|D ]�}i }x<|jD ]2}x,|jD ]"\}} |d k rJ� | |j|i �� qJW q>W xRt|j� �D ]B\}}
x,|D ]} | j|
kr�P q�W t |
|�} |j| � |j| |� q�W q.W |S )Nc s i }� | |� |S )Nr )r3 �base)�
addclosurer r
�closure� s
z)ParserGenerator.make_dfa.<locals>.closurec s> | |kr d S d|| < x$| j D ]\}}|d kr � ||� q W d S )Nr )r% )r3 rN r4 r5 )rO r r
rO � s z,ParserGenerator.make_dfa.<locals>.addclosure)�DFAState�nfasetr% �
setdefaultr$ r& r' �addarc) r r- �finishrP r+ r3 r% Znfastater4 r5 rR �str )rO r
rJ � s"
zParserGenerator.make_dfac
C s� t d|� |g}x�t|�D ]�\}}t d|||kr4dp6d� x^|jD ]T\}}||kr^|j|�} nt|�} |j|� |d kr�t d| � qBt d|| f � qBW qW d S )NzDump of NFA forz Statez(final)� z -> %dz %s -> %d)�print� enumerater% r) r" r' )
r r0 r- rU Ztodor1 r3 r4 r5 �jr r r
�dump_nfa� s
zParserGenerator.dump_nfac C sl t d|� x\t|�D ]P\}}t d||jr,dp.d� x0t|jj� �D ] \}}t d||j|�f � qBW qW d S )NzDump of DFA forz Statez(final)rW z %s -> %d)rX rY r* r$ r% r& r) )r r0 r2 r1 r3 r4 r5 r r r
�dump_dfa� s
zParserGenerator.dump_dfac C s~ d}xt|rxd}xft |�D ]Z\}}xPt|d t|��D ]:}|| }||kr4||= x|D ]}|j||� qTW d}P q4W qW qW d S )NTFr )rY �ranger" �
unifystate)r r2 Zchangesr1 Zstate_irZ Zstate_jr3 r r r
rK � s
z ParserGenerator.simplify_dfac C s� | j � \}}| jdkr ||fS t� }t� }|j|� |j|� x6| jdkrt| j� | j � \}}|j|� |j|� q@W ||fS d S )N�|)� parse_altr> �NFAStaterT r )r rL rM ZaaZzzr r r
rI � s
zParserGenerator.parse_rhsc C sP | j � \}}x:| jdks*| jtjtjfkrF| j � \}}|j|� |}qW ||fS )N�(�[)rb rc )�
parse_itemr> rD r r= �STRINGrT )r rL �br. �dr r r
r`
s
zParserGenerator.parse_altc C s� | j dkr>| j� | j� \}}| jtjd� |j|� ||fS | j� \}}| j }|dkr`||fS | j� |j|� |dkr�||fS ||fS d S )Nrc �]�+�*)ri rj )r> r rI rG r rH rT �
parse_atom)r rL rM r> r r r
rd s
zParserGenerator.parse_itemc C s� | j dkr4| j� | j� \}}| jtjd� ||fS | jtjtjfkrpt � }t � }|j
|| j � | j� ||fS | j d| j| j � d S )Nrb �)z+expected (...) or NAME or STRING, got %s/%s) r> r rI rG r rH rD r= re ra rT � raise_error)r rL rM r r r
rk ( s
zParserGenerator.parse_atomc C sD | j |ks |d k r2| j|kr2| jd||| j | j� | j}| j� |S )Nzexpected %s/%s, got %s/%s)rD r> rm r )r rD r> r r r
rG 9 s zParserGenerator.expectc C sJ t | j�}x"|d tjtjfkr,t | j�}q W |\| _| _| _| _| _ d S )Nr )
r5 r r �COMMENT�NLrD r> Zbegin�end�line)r �tupr r r
r A s
zParserGenerator.gettokenc G s^ |r8y || }W n& dj |gttt|�� �}Y nX t|| j| jd | jd | jf��d S )N� r r )�joinr �map�str� SyntaxErrorr rp rq )r �msg�argsr r r
rm H s zParserGenerator.raise_error)N)N)r r r r r6 r, r( r r? r rJ r[ r\ rK rI r` rd rk rG r rm r r r r
r
s$
. $
r c @ s e Zd Zdd� Zddd�ZdS )ra c C s
g | _ d S )N)r% )r r r r
r S s zNFAState.__init__Nc C s | j j||f� d S )N)r% r' )r r5 r4 r r r
rT V s zNFAState.addarc)N)r r r r rT r r r r
ra Q s ra c @ s0 e Zd Zdd� Zdd� Zdd� Zdd� Zd Zd S )
rQ c C s || _ ||k| _i | _d S )N)rR r* r% )r rR �finalr r r
r ] s
zDFAState.__init__c C s || j |< d S )N)r% )r r5 r4 r r r
rT e s zDFAState.addarcc C s. x(| j j� D ]\}}||kr || j |< q W d S )N)r% r& )r �old�newr4 r5 r r r
r^ k s zDFAState.unifystatec C sX | j |j krdS t| j�t|j�kr(dS x*| jj� D ] \}}||jj|�k r4dS q4W dS )NFT)r* r" r% r& �get)r �otherr4 r5 r r r
�__eq__p s zDFAState.__eq__N)r r r r rT r^ r �__hash__r r r r
rQ [ s
rQ � Grammar.txtc C s t | �}|j� S )N)r r6 )r �pr r r
�generate_grammar� s r� N)r� ) rW r r r ZGrammarr �objectr ra rQ r� r r r r
�<module> s I
% __pycache__/literals.cpython-36.pyc 0000644 00000002776 15046736237 0013252 0 ustar 00 3
\O � @ sP d Z ddlZddddddd d
d d d
�
Zdd� Zdd� Zdd� ZedkrLe� dS )z<Safely evaluate Python string literals without using eval().� N��� �
�
� � �'�"�\)
�a�b�f�n�r�t�vr r
r c C s� | j dd�\}}|jd�s t�tj|�}|d k r4|S |jd�r�|dd � }t|�dk rbtd| ��yt|d�}W q� tk
r� td| ��Y q�X n0yt|d�}W n tk
r� td | ��Y nX t|�S )
Nr � r �x� z!invalid hex string escape ('\%s')� � z#invalid octal string escape ('\%s')) �group�
startswith�AssertionError�simple_escapes�get�len�
ValueError�int�chr)�m�all�tail�escZhexes�i� r& �./usr/lib64/python3.6/lib2to3/pgen2/literals.py�escape s$
r( c C s� | j d�s(| j d�s(tt| d d� ���| d }| d d� |d krL|d }| j|�sptt| t|� d � ���t| �dt|� ks�t�| t|�t|� � } tjdt| �S )Nr r
r r � r z)\\(\'|\"|\\|[abfnrtv]|x.{0,2}|[0-7]{1,3}))r r �repr�endswithr �re�subr( )�s�qr& r& r' �
evalString( s ($r0 c C sD x>t d�D ]2} t| �}t|�}t|�}||kr
t| |||� q
W d S )N� )�ranger r* r0 �print)r% �cr. �er& r& r' �test2 s r6 �__main__)�__doc__r, r r( r0 r6 �__name__r& r& r& r' �<module> s
__pycache__/__init__.cpython-36.opt-2.pyc 0000644 00000000203 15046736237 0014111 0 ustar 00 3
\� � @ s d S )N� r r r �./usr/lib64/python3.6/lib2to3/pgen2/__init__.py�<module> s __pycache__/tokenize.cpython-36.pyc 0000644 00000036032 15046736237 0013253 0 ustar 00 3
\NX � = @ s� d Z dZdZddlZddlZddlmZmZ ddlT ddl m
Z
d d
� e e
�D �d d d
g Z [
ye
W n ek
r~ eZ
Y nX dd� Zdd� Zdd� ZdZdZeede � ee� ZdZdZdZdZedd �Zeeeee�Zd Z ed d �ee � Z d e Z ee e �Z ed!e d" �Z ee e e�Z!d#Z"d$Z#d%Z$d&Z%d'Z&ee&d( e&d) �Z'ee&d* e&d+ �Z(ed,d-d.d/d0d1d2d3d4� Z)d5Z*ed6d7�Z+ee)e*e+�Z,ee!e,e(e�Z-ee- Z.ee&d8 ed9d� e&d: ed;d� �Z/edee'�Z0eee0e!e,e/e� Z1e2e3ej4e.e1e$e%f��\Z5Z6Z7Z8ej4e"�ej4e#�e7e8e7e8e7e8e7e8e7e8e7e8e7e8e7e8e7e8e7e8e7e8e7e8e7e8e7e8e7e8e7e8e7e8e7e8e7e8e7e8e7e8ddddddddd<�4Z9i Z:xd�D ]Z;e;e:e;< �q�W i Z<xd�D ]Z;e;e<e;< �q�W d�Z=G d�d�� d�e>�Z?G d�d�� d�e>�Z@d�d�� ZAeAfd�d �ZBd�d�� ZCG d�d�� d��ZDej4d�ejE�ZFej4d�ejE�ZGd�d�� ZHd�d�� ZId�d
� ZJd�d � ZKeLd�k�r�ddlMZMeNeMjO�dk�r|eBePeMjOd �jQ� n eBeMjRjQ� dS )�a� Tokenization help for Python programs.
generate_tokens(readline) is a generator that breaks a stream of
text into Python tokens. It accepts a readline-like method which is called
repeatedly to get the next line of input (or "" for EOF). It generates
5-tuples with these members:
the token type (see token.py)
the token (a string)
the starting (row, column) indices of the token (a 2-tuple of ints)
the ending (row, column) indices of the token (a 2-tuple of ints)
the original line (string)
It is designed to match the working of the Python tokenizer exactly, except
that it produces COMMENT tokens for comments and gives type OP for all
operators
Older entry points
tokenize_loop(readline, tokeneater)
tokenize(readline, tokeneater=printtoken)
are the same, except instead of generating tokens, tokeneater is a callback
function to which the 5 fields described above are passed as 5 arguments,
each time a new token is found.zKa-Ping Yee <ping@lfw.org>z@GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, Skip Montanaro� N)�BOM_UTF8�lookup)�*� )�tokenc C s g | ]}|d dkr|�qS )r �_� )�.0�xr r �./usr/lib64/python3.6/lib2to3/pgen2/tokenize.py�
<listcomp>% s r �tokenize�generate_tokens�
untokenizec G s ddj | � d S )N�(�|�))�join)�choicesr r r �group0 s r c G s t | � d S )Nr )r )r r r r �any1 s r c G s t | � d S )N�?)r )r r r r �maybe2 s r z[ \f\t]*z #[^\r\n]*z\\\r?\nz [a-zA-Z_]\w*z0[bB]_?[01]+(?:_[01]+)*z(0[xX]_?[\da-fA-F]+(?:_[\da-fA-F]+)*[lL]?z 0[oO]?_?[0-7]+(?:_[0-7]+)*[lL]?z[1-9]\d*(?:_\d+)*[lL]?z0[lL]?z[eE][-+]?\d+(?:_\d+)*z \d+(?:_\d+)*\.(?:\d+(?:_\d+)*)?z\.\d+(?:_\d+)*z \d+(?:_\d+)*z\d+(?:_\d+)*[jJ]z[jJ]z[^'\\]*(?:\\.[^'\\]*)*'z[^"\\]*(?:\\.[^"\\]*)*"z%[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''z%[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""z#(?:[uUrRbBfF]|[rR][bB]|[bBuU][rR])?�'''�"""z '[^\n'\\]*(?:\\.[^\n'\\]*)*'z "[^\n"\\]*(?:\\.[^\n"\\]*)*"z\*\*=?z>>=?z<<=?z<>z!=z//=?z->z[+\-*/%&@|^=<>]=?�~z[][(){}]z\r?\nz[:;.,`@]z'[^\n'\\]*(?:\\.[^\n'\\]*)*�'z"[^\n"\\]*(?:\\.[^\n"\\]*)*�")4r r z'''z"""zr'''zr"""zu'''zu"""zb'''zb"""zf'''zf"""zur'''zur"""zbr'''zbr"""zrb'''zrb"""zR'''zR"""zU'''zU"""zB'''zB"""zF'''zF"""zuR'''zuR"""zUr'''zUr"""zUR'''zUR"""zbR'''zbR"""zBr'''zBr"""zBR'''zBR"""zrB'''zrB"""zRb'''zRb"""zRB'''zRB"""�r�R�u�U�f�F�b�B�r'''�r"""�R'''�R"""�u'''�u"""�U'''�U"""�b'''�b"""�B'''�B"""�f'''�f"""�F'''�F"""�ur'''�ur"""�Ur'''�Ur"""�uR'''�uR"""�UR'''�UR"""�br'''�br"""�Br'''�Br"""�bR'''�bR"""�BR'''�BR"""�rb'''�rb"""�Rb'''�Rb"""�rB'''�rB"""�RB'''�RB"""�r'�r"�R'�R"�u'�u"�U'�U"�b'�b"�B'�B"�f'�f"�F'�F"�ur'�ur"�Ur'�Ur"�uR'�uR"�UR'�UR"�br'�br"�Br'�Br"�bR'�bR"�BR'�BR"�rb'�rb"�Rb'�Rb"�rB'�rB"�RB'�RB"� c @ s e Zd ZdS )�
TokenErrorN)�__name__�
__module__� __qualname__r r r r rw � s rw c @ s e Zd ZdS )�StopTokenizingN)rx ry rz r r r r r{ � s r{ c C s4 |\}}|\}}t d||||t| t|�f � d S )Nz%d,%d-%d,%d: %s %s)�print�tok_name�repr) �typer Zxxx_todo_changemeZxxx_todo_changeme1�lineZsrowZscolZerowZecolr r r �
printtoken� s r� c C s( yt | |� W n tk
r" Y nX dS )a:
The tokenize() function accepts two parameters: one representing the
input stream, and one providing an output mechanism for tokenize().
The first parameter, readline, must be a callable object which provides
the same interface as the readline() method of built-in file objects.
Each call to the function should return one line of input as a string.
The second parameter, tokeneater, must also be a callable object. It is
called once for each token, with five arguments, corresponding to the
tuples generated by generate_tokens().
N)�
tokenize_loopr{ )�readline�
tokeneaterr r r r
� s
c C s xt | �D ] }||� q
W d S )N)r )r� r� Z
token_infor r r r� � s r� c @ s, e Zd Zdd� Zdd� Zdd� Zdd� Zd S )
� Untokenizerc C s g | _ d| _d| _d S )Nr r )�tokens�prev_row�prev_col)�selfr r r �__init__� s zUntokenizer.__init__c C s8 |\}}|| j kst�|| j }|r4| jjd| � d S )N� )r� �AssertionErrorr� r� �append)r� �start�row�col�
col_offsetr r r �add_whitespace� s
zUntokenizer.add_whitespacec C s� xv|D ]n}t |�dkr$| j||� P |\}}}}}| j|� | jj|� |\| _| _|ttfkr| jd7 _d| _qW dj | j�S )N� r r � )
�len�compatr� r� r� r� r� �NEWLINE�NLr )r� �iterable�t�tok_typer r� �endr� r r r r � s
zUntokenizer.untokenizec C s� d}g }| j j}|\}}|ttfkr,|d7 }|ttfkr<d}x�|D ]�}|d d� \}}|ttttfkrn|d7 }|tkr�|j|� qBn>|t kr�|j
� qBn*|ttfkr�d}n|r�|r�||d � d}||� qBW d S )NFr� Tr� r ���) r� r� �NAME�NUMBERr� r� �ASYNC�AWAIT�INDENT�DEDENT�pop) r� r r� � startline�indents� toks_append�toknum�tokval�tokr r r r� � s0
zUntokenizer.compatN)rx ry rz r� r� r r� r r r r r� � s r� z&^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)s ^[ \t\f]*(?:[#\r\n]|$)c C sH | dd� j � jdd�}|dks*|jd�r.dS |d
ks@|jd�rDdS | S )z(Imitates get_normal_name in tokenizer.c.N� r �-zutf-8zutf-8-�latin-1�
iso-8859-1� iso-latin-1�latin-1-� iso-8859-1-� iso-latin-1-)r� r� r� )r� r� r� )�lower�replace�
startswith)�orig_enc�encr r r �_get_normal_name s
r� c s� d� d}d}�fdd�}� fdd�}|� }|j t�rHd� |d d� }d
}|sT|g fS ||�}|rj||gfS tj|�s~||gfS |� }|s�||gfS ||�}|r�|||gfS |||gfS ) a
The detect_encoding() function is used to detect the encoding that should
be used to decode a Python source file. It requires one argument, readline,
in the same way as the tokenize() generator.
It will call readline a maximum of twice, and return the encoding used
(as a string) and a list of any lines (left as bytes) it has read
in.
It detects the encoding from the presence of a utf-8 bom or an encoding
cookie as specified in pep-0263. If both a bom and a cookie are present, but
disagree, a SyntaxError will be raised. If the encoding cookie is an invalid
charset, raise a SyntaxError. Note that if a utf-8 bom is found,
'utf-8-sig' is returned.
If no encoding is specified, then the default of 'utf-8' will be returned.
FNzutf-8c s" y� � S t k
r t� S X d S )N)�
StopIteration�bytesr )r� r r � read_or_stop s z%detect_encoding.<locals>.read_or_stopc s� y| j d�}W n tk
r" d S X tj|�}|s6d S t|jd��}y t|�}W n tk
rp td| ��Y nX � r�|j dkr�td��|d7 }|S )N�asciir zunknown encoding: zutf-8zencoding problem: utf-8z-sig)
�decode�UnicodeDecodeError� cookie_re�matchr� r r � LookupError� SyntaxError�name)r� � line_stringr� �encoding�codec)� bom_foundr r � find_cookie& s"
z$detect_encoding.<locals>.find_cookieT� z utf-8-sig)r� r �blank_rer� )r� r� �defaultr� r� �first�secondr )r� r� r �detect_encoding s0
r� c C s t � }|j| �S )a� Transform tokens back into Python source code.
Each element returned by the iterable must be a token sequence
with at least two elements, a token number and token value. If
only two tokens are passed, the resulting output is poor.
Round-trip invariant for full input:
Untokenized source will match input source exactly
Round-trip invariant for limited intput:
# Output text will tokenize the back to the input
t1 = [tok[:2] for tok in generate_tokens(f.readline)]
newcode = untokenize(t1)
readline = iter(newcode.splitlines(1)).next
t2 = [tok[:2] for tokin generate_tokens(readline)]
assert t1 == t2
)r� r )r� �utr r r r T s c ! c s� d } }}t jd d }}d \}}d}dg} d}
d} d} d}
�xy
| � }W n tk
rd d}Y nX |d }dt|� }}|�rF|s�td|��|j|�}|r�|jd� }}t||d|� |||f|| fV d!\}}d}nd|�r0|d"d� d
k�r0|d#d� d k�r0t|| ||t|�f|fV d}d}qBn|| }|| }qB�nF|dk�rt| �rt|�s`P d}xf||k �r�|| d
k�r�|d }n6|| dk�r�|t d t }n|| dk�r�d}nP |d }�qfW ||k�r�P |
�r�|
V d}
|| dk�r�|| dk�rh||d� j
d�}|t|� }t |||f||t|� f|fV t ||d� ||f|t|�f|fV qBt t f|| dk ||d� ||f|t|�f|fV qB|| d$ k�r�| j
|� t|d|� |df||f|fV xt|| d% k �rJ|| k�rtdd|||f��| dd&� } | �r.| | d' k�r.d} d}
d} td||f||f|fV �q�W | �r�|
�r�| | d( k�r�d} d}
d} n|�s�td|df��d}�x�||k �rJtj||�}|�r |jd�\}}||f||f| }}}|||� || } } | |k�s| dk�r| dk�rt| |||fV �qF| dk�rft} |dk�r8t } n
| �rBd}
|
�rR|
V d}
| | |||fV �qF| dk�r�| jd� �s�t�|
�r�|
V d}
t | |||fV �qF| tk�r$t| }|j||�}|�r|jd�}|||� } |
�r�|
V d}
t| |||f|fV n||f}||d� }|}P �qF| tk�sR| dd � tk�sR| dd � tk�r�| d) dk�r�||f}t| �p�t| d �p�t| d }||d� d }}|}P n |
�r�|
V d}
t| |||fV �qF| |k�r�| d*k�r| �r| dk�r�tnt| |||fV �q�t | |||f} | dk�r.|
�r.| }
�q�| dk�r�|
�r�|
d t k�r�|
d dk�r�d} | d+ } t|
d |
d |
d |
d fV d}
|
�r�|
V d}
| V nz| d k�r�|
�r�|
V d}
t | |||f|fV d}nF| d k�r�|d }n| d k�r�|d }|
�r |
V d}
t | |||fV n(t|| ||f||d f|fV |d }�q�W qBW |
�r`|
V d}
x.| dd� D ] } td|df|dfdfV �qnW t d|df|dfdfV dS ),aT
The generate_tokens() generator requires one argument, readline, which
must be a callable object which provides the same interface as the
readline() method of built-in file objects. Each call to the function
should return one line of input as a string. Alternately, readline
can be a callable function terminating with StopIteration:
readline = open(myfile).next # Example of alternate readline
The generator produces 5-tuples with these members: the token type; the
token string; a 2-tuple (srow, scol) of ints specifying the row and
column where the token begins in the source; a 2-tuple (erow, ecol) of
ints specifying the row and column where the token ends in the source;
and the line on which the token was found. The line passed is the
logical line; continuation lines are included.
r r �
0123456789r� NFr zEOF in multi-line stringr� z\
r� z\
r� � � z#
�#z
z3unindent does not match any outer indentation levelz
<tokenize>zEOF in multi-line statement�.T�
�async�await�def� �\z([{z)]})r� r )r� r ������r� r� r� r� r� r� )r� r� r� ) �stringZ
ascii_lettersr� r� rw r� r� �STRING�
ERRORTOKEN�tabsize�rstrip�COMMENTr� r� r� �IndentationErrorr� �
pseudoprog�spanr� r� �endswithr� �
triple_quoted�endprogs�
single_quotedr� r� r� �OP� ENDMARKER)!r� �lnum�parenlev� continuedZ namechars�numchars�contstr�needcont�contliner� �stashed� async_def�async_def_indent� async_def_nlr� �pos�max�strstart�endprog�endmatchr� �column�
comment_token�nl_pos� pseudomatchr� �spos�eposr �initial�newliner� �indentr r r r i sr
*
�__main__)*r r r& r' r( r) r* r+ r, r- r. r/ r0 r1 r2 r3 r4 r5 r6 r7 r8 r9 r: r; r< r= r> r? r@ rA rB rC rD rE rF rG rH rI rJ rK rL rM )*r r rN rO rP rQ rR rS rT rU rV rW rX rY rZ r[ r\ r] r^ r_ r` ra rb rc rd re rf rg rh ri rj rk rl rm rn ro rp rq rr rs rt ru )S�__doc__�
__author__� __credits__r� �re�codecsr r Zlib2to3.pgen2.tokenr� r �dir�__all__r� � NameError�strr r r �
Whitespace�Comment�Ignore�Name� Binnumber� Hexnumber� Octnumber� Decnumber� Intnumber�Exponent�
Pointfloat�Expfloat� Floatnumber�
Imagnumber�Number�Single�Double�Single3�Double3Z
_litprefix�Triple�String�Operator�Bracket�Special�Funny�
PlainToken�Token�ContStr� PseudoExtras� PseudoToken�list�map�compileZ tokenprogr� Z single3progZ double3progr� r� r� r� r� � Exceptionrw r{ r� r
r� r� �ASCIIr� r� r� r� r r rx �sysr� �argv�openr� �stdinr r r r �<module> s�
8 I c
__pycache__/literals.cpython-36.opt-1.pyc 0000644 00000002502 15046736237 0014174 0 ustar 00 3
\O � @ sP d Z ddlZddddddd d
d d d
�
Zdd� Zdd� Zdd� ZedkrLe� dS )z<Safely evaluate Python string literals without using eval().� N��� �
�
� � �'�"�\)
�a�b�f�n�r�t�vr r
r c C s� | j dd�\}}tj|�}|d k r&|S |jd�r�|dd � }t|�dk rTtd| ��yt|d�}W q� tk
r� td| ��Y q�X n0yt|d�}W n tk
r� td| ��Y nX t|�S ) Nr � �x� z!invalid hex string escape ('\%s')� � z#invalid octal string escape ('\%s'))�group�simple_escapes�get�
startswith�len�
ValueError�int�chr)�m�all�tail�escZhexes�i� r% �./usr/lib64/python3.6/lib2to3/pgen2/literals.py�escape s"
r' c C sH | d }| d d� |d kr$|d }| t |�t |� � } tjdt| �S )Nr � z)\\(\'|\"|\\|[abfnrtv]|x.{0,2}|[0-7]{1,3}))r �re�subr' )�s�qr% r% r& �
evalString( s
r- c C sD x>t d�D ]2} t| �}t|�}t|�}||kr
t| |||� q
W d S )N� )�ranger �reprr- �print)r$ �cr+ �er% r% r& �test2 s r4 �__main__)�__doc__r) r r' r- r4 �__name__r% r% r% r& �<module> s
__pycache__/grammar.cpython-36.pyc 0000644 00000015621 15046736237 0013052 0 ustar 00 3
\� � @ sx d Z ddlZddlZddlmZmZ G dd� de�Zdd� Zd Z i Z
x.e j � D ]"Z e rNe j
� \ZZeee�e
e< qNW dS )
a� This module defines the data structures used to represent a grammar.
These are a bit arcane because they are derived from the data
structures used by Python's 'pgen' parser generator.
There's also a table here mapping operators to their names in the
token module; the Python tokenize module reports all operators as the
fallback token code OP, but the parser needs the actual token code.
� N� )�token�tokenizec @ s@ e Zd ZdZdd� Zdd� Zdd� Zdd � Zd
d � Zd d
� Z dS )�Grammara� Pgen parsing tables conversion class.
Once initialized, this class supplies the grammar tables for the
parsing engine implemented by parse.py. The parsing engine
accesses the instance variables directly. The class here does not
provide initialization of the tables; several subclasses exist to
do this (see the conv and pgen modules).
The load() method reads the tables from a pickle file, which is
much faster than the other ways offered by subclasses. The pickle
file is written by calling dump() (after loading the grammar
tables using a subclass). The report() method prints a readable
representation of the tables to stdout, for debugging.
The instance variables are as follows:
symbol2number -- a dict mapping symbol names to numbers. Symbol
numbers are always 256 or higher, to distinguish
them from token numbers, which are between 0 and
255 (inclusive).
number2symbol -- a dict mapping numbers to symbol names;
these two are each other's inverse.
states -- a list of DFAs, where each DFA is a list of
states, each state is a list of arcs, and each
arc is a (i, j) pair where i is a label and j is
a state number. The DFA number is the index into
this list. (This name is slightly confusing.)
Final states are represented by a special arc of
the form (0, j) where j is its own state number.
dfas -- a dict mapping symbol numbers to (DFA, first)
pairs, where DFA is an item from the states list
above, and first is a set of tokens that can
begin this grammar rule (represented by a dict
whose values are always 1).
labels -- a list of (x, y) pairs where x is either a token
number or a symbol number, and y is either None
or a string; the strings are keywords. The label
number is the index in this list; label numbers
are used to mark state transitions (arcs) in the
DFAs.
start -- the number of the grammar's start symbol.
keywords -- a dict mapping keyword strings to arc labels.
tokens -- a dict mapping token numbers to arc labels.
c C s< i | _ i | _g | _i | _dg| _i | _i | _i | _d| _d S )Nr �EMPTY� )r r ) �
symbol2number�
number2symbol�states�dfas�labels�keywords�tokens� symbol2label�start)�self� r �-/usr/lib64/python3.6/lib2to3/pgen2/grammar.py�__init__M s zGrammar.__init__c C s2 t |d�� }t| j�}tj||d� W dQ R X dS )a� Dump the grammar tables to a pickle file.
dump() recursively changes all dict to OrderedDict, so the pickled file
is not exactly the same as what was passed in to dump(). load() uses the
pickled file to create the tables, but only changes OrderedDict to dict
at the top level; it does not recursively change OrderedDict to dict.
So, the loaded tables are different from the original tables that were
passed to load() in that some of the OrderedDict (from the pickled file)
are not changed back to dict. For parsing, this has no effect on
performance because OrderedDict uses dict's __getitem__ with nothing in
between.
�wb� N)�open�_make_deterministic�__dict__�pickle�dump)r �filename�f�dr r r r X s
z Grammar.dumpc C s0 t |d��}tj|�}W dQ R X | jj|� dS )z+Load the grammar tables from a pickle file.�rbN)r r �loadr �update)r r r r r r r r i s z Grammar.loadc C s | j jtj|�� dS )z3Load the grammar tables from a pickle bytes object.N)r r! r �loads)r Zpklr r r r" o s z
Grammar.loadsc C sX | j � }x"dD ]}t||t| |�j� � qW | jdd� |_| jdd� |_| j|_|S ) z#
Copy the grammar.
r r r r
r r N)r r r r
r r )� __class__�setattr�getattr�copyr r
r )r �newZ dict_attrr r r r& s s z Grammar.copyc C sv ddl m } td� || j� td� || j� td� || j� td� || j� td� || j� td| j� d S )
z:Dump the grammar tables to standard output, for debugging.r )�pprintZs2nZn2sr
r r r N)r( �printr r r
r r r )r r( r r r �report� s
zGrammar.reportN)
�__name__�
__module__� __qualname__�__doc__r r r r" r& r* r r r r r s 4
r c C s^ t | t�r&tjtdd� | j� D ���S t | t�r>dd� | D �S t | t�rZtdd� | D ��S | S )Nc s s | ]\}}|t |�fV qd S )N)r )�.0�k�vr r r � <genexpr>� s z&_make_deterministic.<locals>.<genexpr>c S s g | ] }t |��qS r )r )r/ �er r r �
<listcomp>� s z'_make_deterministic.<locals>.<listcomp>c s s | ]}t |�V qd S )N)r )r/ r3 r r r r2 � s )�
isinstance�dict� collections� OrderedDict�sorted�items�list�tuple)�topr r r r � s
r a
( LPAR
) RPAR
[ LSQB
] RSQB
: COLON
, COMMA
; SEMI
+ PLUS
- MINUS
* STAR
/ SLASH
| VBAR
& AMPER
< LESS
> GREATER
= EQUAL
. DOT
% PERCENT
` BACKQUOTE
{ LBRACE
} RBRACE
@ AT
@= ATEQUAL
== EQEQUAL
!= NOTEQUAL
<> NOTEQUAL
<= LESSEQUAL
>= GREATEREQUAL
~ TILDE
^ CIRCUMFLEX
<< LEFTSHIFT
>> RIGHTSHIFT
** DOUBLESTAR
+= PLUSEQUAL
-= MINEQUAL
*= STAREQUAL
/= SLASHEQUAL
%= PERCENTEQUAL
&= AMPEREQUAL
|= VBAREQUAL
^= CIRCUMFLEXEQUAL
<<= LEFTSHIFTEQUAL
>>= RIGHTSHIFTEQUAL
**= DOUBLESTAREQUAL
// DOUBLESLASH
//= DOUBLESLASHEQUAL
-> RARROW
)r. r7 r � r r �objectr r Z opmap_rawZopmap�
splitlines�line�split�op�namer% r r r r �<module>
s y= __pycache__/tokenize.cpython-36.opt-1.pyc 0000644 00000035722 15046736237 0014217 0 ustar 00 3
\NX � = @ s� d Z dZdZddlZddlZddlmZmZ ddlT ddl m
Z
d d
� e e
�D �d d d
g Z [
ye
W n ek
r~ eZ
Y nX dd� Zdd� Zdd� ZdZdZeede � ee� ZdZdZdZdZedd �Zeeeee�Zd Z ed d �ee � Z d e Z ee e �Z ed!e d" �Z ee e e�Z!d#Z"d$Z#d%Z$d&Z%d'Z&ee&d( e&d) �Z'ee&d* e&d+ �Z(ed,d-d.d/d0d1d2d3d4� Z)d5Z*ed6d7�Z+ee)e*e+�Z,ee!e,e(e�Z-ee- Z.ee&d8 ed9d� e&d: ed;d� �Z/edee'�Z0eee0e!e,e/e� Z1e2e3ej4e.e1e$e%f��\Z5Z6Z7Z8ej4e"�ej4e#�e7e8e7e8e7e8e7e8e7e8e7e8e7e8e7e8e7e8e7e8e7e8e7e8e7e8e7e8e7e8e7e8e7e8e7e8e7e8e7e8e7e8ddddddddd<�4Z9i Z:xd�D ]Z;e;e:e;< �q�W i Z<xd�D ]Z;e;e<e;< �q�W d�Z=G d�d�� d�e>�Z?G d�d�� d�e>�Z@d�d�� ZAeAfd�d �ZBd�d�� ZCG d�d�� d��ZDej4d�ejE�ZFej4d�ejE�ZGd�d�� ZHd�d�� ZId�d
� ZJd�d � ZKeLd�k�r�ddlMZMeNeMjO�dk�r|eBePeMjOd �jQ� n eBeMjRjQ� dS )�a� Tokenization help for Python programs.
generate_tokens(readline) is a generator that breaks a stream of
text into Python tokens. It accepts a readline-like method which is called
repeatedly to get the next line of input (or "" for EOF). It generates
5-tuples with these members:
the token type (see token.py)
the token (a string)
the starting (row, column) indices of the token (a 2-tuple of ints)
the ending (row, column) indices of the token (a 2-tuple of ints)
the original line (string)
It is designed to match the working of the Python tokenizer exactly, except
that it produces COMMENT tokens for comments and gives type OP for all
operators
Older entry points
tokenize_loop(readline, tokeneater)
tokenize(readline, tokeneater=printtoken)
are the same, except instead of generating tokens, tokeneater is a callback
function to which the 5 fields described above are passed as 5 arguments,
each time a new token is found.zKa-Ping Yee <ping@lfw.org>z@GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, Skip Montanaro� N)�BOM_UTF8�lookup)�*� )�tokenc C s g | ]}|d dkr|�qS )r �_� )�.0�xr r �./usr/lib64/python3.6/lib2to3/pgen2/tokenize.py�
<listcomp>% s r �tokenize�generate_tokens�
untokenizec G s ddj | � d S )N�(�|�))�join)�choicesr r r �group0 s r c G s t | � d S )Nr )r )r r r r �any1 s r c G s t | � d S )N�?)r )r r r r �maybe2 s r z[ \f\t]*z #[^\r\n]*z\\\r?\nz [a-zA-Z_]\w*z0[bB]_?[01]+(?:_[01]+)*z(0[xX]_?[\da-fA-F]+(?:_[\da-fA-F]+)*[lL]?z 0[oO]?_?[0-7]+(?:_[0-7]+)*[lL]?z[1-9]\d*(?:_\d+)*[lL]?z0[lL]?z[eE][-+]?\d+(?:_\d+)*z \d+(?:_\d+)*\.(?:\d+(?:_\d+)*)?z\.\d+(?:_\d+)*z \d+(?:_\d+)*z\d+(?:_\d+)*[jJ]z[jJ]z[^'\\]*(?:\\.[^'\\]*)*'z[^"\\]*(?:\\.[^"\\]*)*"z%[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''z%[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""z#(?:[uUrRbBfF]|[rR][bB]|[bBuU][rR])?�'''�"""z '[^\n'\\]*(?:\\.[^\n'\\]*)*'z "[^\n"\\]*(?:\\.[^\n"\\]*)*"z\*\*=?z>>=?z<<=?z<>z!=z//=?z->z[+\-*/%&@|^=<>]=?�~z[][(){}]z\r?\nz[:;.,`@]z'[^\n'\\]*(?:\\.[^\n'\\]*)*�'z"[^\n"\\]*(?:\\.[^\n"\\]*)*�")4r r z'''z"""zr'''zr"""zu'''zu"""zb'''zb"""zf'''zf"""zur'''zur"""zbr'''zbr"""zrb'''zrb"""zR'''zR"""zU'''zU"""zB'''zB"""zF'''zF"""zuR'''zuR"""zUr'''zUr"""zUR'''zUR"""zbR'''zbR"""zBr'''zBr"""zBR'''zBR"""zrB'''zrB"""zRb'''zRb"""zRB'''zRB"""�r�R�u�U�f�F�b�B�r'''�r"""�R'''�R"""�u'''�u"""�U'''�U"""�b'''�b"""�B'''�B"""�f'''�f"""�F'''�F"""�ur'''�ur"""�Ur'''�Ur"""�uR'''�uR"""�UR'''�UR"""�br'''�br"""�Br'''�Br"""�bR'''�bR"""�BR'''�BR"""�rb'''�rb"""�Rb'''�Rb"""�rB'''�rB"""�RB'''�RB"""�r'�r"�R'�R"�u'�u"�U'�U"�b'�b"�B'�B"�f'�f"�F'�F"�ur'�ur"�Ur'�Ur"�uR'�uR"�UR'�UR"�br'�br"�Br'�Br"�bR'�bR"�BR'�BR"�rb'�rb"�Rb'�Rb"�rB'�rB"�RB'�RB"� c @ s e Zd ZdS )�
TokenErrorN)�__name__�
__module__� __qualname__r r r r rw � s rw c @ s e Zd ZdS )�StopTokenizingN)rx ry rz r r r r r{ � s r{ c C s4 |\}}|\}}t d||||t| t|�f � d S )Nz%d,%d-%d,%d: %s %s)�print�tok_name�repr) �typer Zxxx_todo_changemeZxxx_todo_changeme1�lineZsrowZscolZerowZecolr r r �
printtoken� s r� c C s( yt | |� W n tk
r" Y nX dS )a:
The tokenize() function accepts two parameters: one representing the
input stream, and one providing an output mechanism for tokenize().
The first parameter, readline, must be a callable object which provides
the same interface as the readline() method of built-in file objects.
Each call to the function should return one line of input as a string.
The second parameter, tokeneater, must also be a callable object. It is
called once for each token, with five arguments, corresponding to the
tuples generated by generate_tokens().
N)�
tokenize_loopr{ )�readline�
tokeneaterr r r r
� s
c C s xt | �D ] }||� q
W d S )N)r )r� r� Z
token_infor r r r� � s r� c @ s, e Zd Zdd� Zdd� Zdd� Zdd� Zd S )
� Untokenizerc C s g | _ d| _d| _d S )Nr r )�tokens�prev_row�prev_col)�selfr r r �__init__� s zUntokenizer.__init__c C s* |\}}|| j }|r&| jjd| � d S )N� )r� r� �append)r� �start�row�col�
col_offsetr r r �add_whitespace� s
zUntokenizer.add_whitespacec C s� xv|D ]n}t |�dkr$| j||� P |\}}}}}| j|� | jj|� |\| _| _|ttfkr| jd7 _d| _qW dj | j�S )N� r r � )
�len�compatr� r� r� r� r� �NEWLINE�NLr )r� �iterable�t�tok_typer r� �endr� r r r r � s
zUntokenizer.untokenizec C s� d}g }| j j}|\}}|ttfkr,|d7 }|ttfkr<d}x�|D ]�}|d d� \}}|ttttfkrn|d7 }|tkr�|j|� qBn>|t kr�|j
� qBn*|ttfkr�d}n|r�|r�||d � d}||� qBW d S )NFr� Tr� r ���) r� r� �NAME�NUMBERr� r� �ASYNC�AWAIT�INDENT�DEDENT�pop) r� r r� � startline�indents� toks_append�toknum�tokval�tokr r r r� � s0
zUntokenizer.compatN)rx ry rz r� r� r r� r r r r r� � s r� z&^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)s ^[ \t\f]*(?:[#\r\n]|$)c C sH | dd� j � jdd�}|dks*|jd�r.dS |d
ks@|jd�rDdS | S )z(Imitates get_normal_name in tokenizer.c.N� r �-zutf-8zutf-8-�latin-1�
iso-8859-1� iso-latin-1�latin-1-� iso-8859-1-� iso-latin-1-)r� r� r� )r� r� r� )�lower�replace�
startswith)�orig_enc�encr r r �_get_normal_name s
r� c s� d� d}d}�fdd�}� fdd�}|� }|j t�rHd� |d d� }d
}|sT|g fS ||�}|rj||gfS tj|�s~||gfS |� }|s�||gfS ||�}|r�|||gfS |||gfS ) a
The detect_encoding() function is used to detect the encoding that should
be used to decode a Python source file. It requires one argument, readline,
in the same way as the tokenize() generator.
It will call readline a maximum of twice, and return the encoding used
(as a string) and a list of any lines (left as bytes) it has read
in.
It detects the encoding from the presence of a utf-8 bom or an encoding
cookie as specified in pep-0263. If both a bom and a cookie are present, but
disagree, a SyntaxError will be raised. If the encoding cookie is an invalid
charset, raise a SyntaxError. Note that if a utf-8 bom is found,
'utf-8-sig' is returned.
If no encoding is specified, then the default of 'utf-8' will be returned.
FNzutf-8c s" y� � S t k
r t� S X d S )N)�
StopIteration�bytesr )r� r r � read_or_stop s z%detect_encoding.<locals>.read_or_stopc s� y| j d�}W n tk
r" d S X tj|�}|s6d S t|jd��}y t|�}W n tk
rp td| ��Y nX � r�|j dkr�td��|d7 }|S )N�asciir zunknown encoding: zutf-8zencoding problem: utf-8z-sig)
�decode�UnicodeDecodeError� cookie_re�matchr� r r � LookupError� SyntaxError�name)r� � line_stringr� �encoding�codec)� bom_foundr r � find_cookie& s"
z$detect_encoding.<locals>.find_cookieT� z utf-8-sig)r� r �blank_rer� )r� r� �defaultr� r� �first�secondr )r� r� r �detect_encoding s0
r� c C s t � }|j| �S )a� Transform tokens back into Python source code.
Each element returned by the iterable must be a token sequence
with at least two elements, a token number and token value. If
only two tokens are passed, the resulting output is poor.
Round-trip invariant for full input:
Untokenized source will match input source exactly
Round-trip invariant for limited intput:
# Output text will tokenize the back to the input
t1 = [tok[:2] for tok in generate_tokens(f.readline)]
newcode = untokenize(t1)
readline = iter(newcode.splitlines(1)).next
t2 = [tok[:2] for tokin generate_tokens(readline)]
assert t1 == t2
)r� r )r� �utr r r r T s c ! c s� d } }}t jd d }}d \}}d}dg} d}
d} d} d}
�x�y
| � }W n tk
rd d}Y nX |d }dt|� }}|�rF|s�td|��|j|�}|r�|jd� }}t||d|� |||f|| fV d!\}}d}nd|�r0|d"d� d
k�r0|d#d� d k�r0t|| ||t|�f|fV d}d}qBn|| }|| }qB�nF|dk�rt| �rt|�s`P d}xf||k �r�|| d
k�r�|d }n6|| dk�r�|t d t }n|| dk�r�d}nP |d }�qfW ||k�r�P |
�r�|
V d}
|| dk�r�|| dk�rh||d� j
d�}|t|� }t |||f||t|� f|fV t ||d� ||f|t|�f|fV qBt t f|| dk ||d� ||f|t|�f|fV qB|| d$ k�r�| j
|� t|d|� |df||f|fV xt|| d% k �rJ|| k�rtdd|||f��| dd&� } | �r.| | d' k�r.d} d}
d} td||f||f|fV �q�W | �r�|
�r�| | d( k�r�d} d}
d} n|�s�td|df��d}�x�||k �r8tj||�}|�r |jd�\}}||f||f| }}}|||� || } } | |k�s| dk�r| dk�rt| |||fV �q4| dk�rft} |dk�r8t } n
| �rBd}
|
�rR|
V d}
| | |||fV �q4| dk�r�|
�r�|
V d}
t | |||fV �q4| tk�rt| }|j||�}|�r�|jd�}|||� } |
�r�|
V d}
t| |||f|fV n||f}||d� }|}P �q4| tk�s@| dd � tk�s@| dd � tk�r�| d) dk�r�||f}t| �pxt| d �pxt| d }||d� d }}|}P n |
�r�|
V d}
t| |||fV �q4| |k�r�| d*k�r�| �r�| dk�r�tnt| |||fV �q�t| |||f} | dk�r |
�r | }
�q�| dk�rx|
�rx|
d tk�rx|
d dk�rxd} | d+ } t|
d |
d |
d |
d fV d}
|
�r�|
V d}
| V nz| d k�r�|
�r�|
V d}
t | |||f|fV d}nF| d k�r�|d }n| d k�r�|d }|
�r�|
V d}
t| |||fV n(t|| ||f||d f|fV |d }�q�W qBW |
�rN|
V d}
x.| dd� D ] } td|df|dfdfV �q\W t d|df|dfdfV dS ),aT
The generate_tokens() generator requires one argument, readline, which
must be a callable object which provides the same interface as the
readline() method of built-in file objects. Each call to the function
should return one line of input as a string. Alternately, readline
can be a callable function terminating with StopIteration:
readline = open(myfile).next # Example of alternate readline
The generator produces 5-tuples with these members: the token type; the
token string; a 2-tuple (srow, scol) of ints specifying the row and
column where the token begins in the source; a 2-tuple (erow, ecol) of
ints specifying the row and column where the token ends in the source;
and the line on which the token was found. The line passed is the
logical line; continuation lines are included.
r r �
0123456789r� NFr zEOF in multi-line stringr� z\
r� z\
r� � � z#
�#z
z3unindent does not match any outer indentation levelz
<tokenize>zEOF in multi-line statement�.T�
�async�await�def� �\z([{z)]})r� r )r� r ������r� r� r� r� r� r� )r� r� r� ) �stringZ
ascii_lettersr� r� rw r� r� �STRING�
ERRORTOKEN�tabsize�rstrip�COMMENTr� r� r� �IndentationErrorr� �
pseudoprog�spanr� r� �
triple_quoted�endprogs�
single_quotedr� r� r� �OP� ENDMARKER)!r� �lnum�parenlev� continuedZ namechars�numchars�contstr�needcont�contliner� �stashed� async_def�async_def_indent� async_def_nlr� �pos�max�strstart�endprog�endmatchr� �column�
comment_token�nl_pos� pseudomatchr� �spos�eposr �initial�newliner� �indentr r r r i sp
*
�__main__)*r r r& r' r( r) r* r+ r, r- r. r/ r0 r1 r2 r3 r4 r5 r6 r7 r8 r9 r: r; r< r= r> r? r@ rA rB rC rD rE rF rG rH rI rJ rK rL rM )*r r rN rO rP rQ rR rS rT rU rV rW rX rY rZ r[ r\ r] r^ r_ r` ra rb rc rd re rf rg rh ri rj rk rl rm rn ro rp rq rr rs rt ru )S�__doc__�
__author__� __credits__r� �re�codecsr r Zlib2to3.pgen2.tokenr� r �dir�__all__r� � NameError�strr r r �
Whitespace�Comment�Ignore�Name� Binnumber� Hexnumber� Octnumber� Decnumber� Intnumber�Exponent�
Pointfloat�Expfloat� Floatnumber�
Imagnumber�Number�Single�Double�Single3�Double3Z
_litprefix�Triple�String�Operator�Bracket�Special�Funny�
PlainToken�Token�ContStr� PseudoExtras� PseudoToken�list�map�compileZ tokenprogr� Z single3progZ double3progr� r� r� r� r� � Exceptionrw r{ r� r
r� r� �ASCIIr� r� r� r� r r rx �sysr� �argv�openr� �stdinr r r r �<module> s�
8 I c
__pycache__/conv.cpython-36.pyc 0000644 00000015570 15046736237 0012374 0 ustar 00 3
\�% � @ s2 d Z ddlZddlmZmZ G dd� dej�ZdS )a� Convert graminit.[ch] spit out by pgen to Python code.
Pgen is the Python parser generator. It is useful to quickly create a
parser from a grammar file in Python's grammar notation. But I don't
want my parsers to be written in C (yet), so I'm translating the
parsing tables to Python data structures and writing a Python parse
engine.
Note that the token numbers are constants determined by the standard
Python tokenizer. The standard token module defines these numbers and
their names (the names are not used much). The token numbers are
hardcoded into the Python tokenizer and into pgen. A Python
implementation of the Python tokenizer is also available, in the
standard tokenize module.
On the other hand, symbol numbers (representing the grammar's
non-terminals) are assigned by pgen based on the actual grammar
input.
Note: this module is pretty much obsolete; the pgen module generates
equivalent grammar tables directly from the Grammar.txt input file
without having to invoke the Python pgen C program.
� N)�grammar�tokenc @ s0 e Zd ZdZdd� Zdd� Zdd� Zdd � Zd
S ) � Convertera2 Grammar subclass that reads classic pgen output files.
The run() method reads the tables as produced by the pgen parser
generator, typically contained in two C files, graminit.h and
graminit.c. The other methods are for internal use only.
See the base class for more documentation.
c C s | j |� | j|� | j� dS )z<Load the grammar tables from the text files written by pgen.N)�parse_graminit_h�parse_graminit_c�
finish_off)�selfZ
graminit_hZ
graminit_c� r �*/usr/lib64/python3.6/lib2to3/pgen2/conv.py�run/ s
z
Converter.runc C s� y t |�}W n0 tk
r< } ztd||f � dS d}~X nX i | _i | _d}x�|D ]�}|d7 }tjd|�}| r�|j� r�td|||j� f � qT|j� \}}t |�}|| jks�t
�|| jks�t
�|| j|< || j|< qTW dS ) z�Parse the .h file written by pgen. (Internal)
This file is a sequence of #define statements defining the
nonterminals of the grammar as numbers. We build two tables
mapping the numbers to names and back.
zCan't open %s: %sFNr � z^#define\s+(\w+)\s+(\d+)$z%s(%s): can't parse %sT) �open�OSError�print�
symbol2number�
number2symbol�re�match�strip�groups�int�AssertionError) r �filename�f�err�lineno�line�mo�symbol�numberr r r
r 5 s*
zConverter.parse_graminit_hc ! C sp y t |�}W n0 tk
r< } ztd||f � dS d}~X nX d}|d t|� }}|dksht||f��|d t|� }}|dks�t||f��|d t|� }}i }g }�x*|jd��r�x�|jd��r�tjd |�}|s�t||f��tt t
|j � ��\} }
} g } xft | �D ]Z}
|d t|� }}tjd
|�}|�s<t||f��tt t
|j � ��\}}| j
||f� �qW |d t|� }}|d k�s�t||f��| || |
f< |d t|� }}q�W tjd |�}|�s�t||f��tt t
|j � ��\}}|t|�k�s�t||f��g }x�t |�D ]~}
|d t|� }}tjd
|�}|�s@t||f��tt t
|j � ��\} } }
|| |
f } | t| �k�s~t||f��|j
| � �q W |j
|� |d t|� }}|d k�s�t||f��|d t|� }}q�W || _i }tjd|�}|�s�t||f��t
|jd��}�x�t |�D �]r}|d t|� }}tjd|�}|�sNt||f��|jd�}tt t
|jdddd���\}}}}| j| |k�s�t||f��| j| |k�s�t||f��|dk�s�t||f��|| }|t|�k�s�t||f��|d t|� }}tjd|�}|�st||f��i }t|jd��}xPt|�D ]D\}} t| �} x0t d�D ]$}| d|> @ �rPd||d | < �qPW �q6W ||f||< �qW |d t|� }}|d k�s�t||f��|| _g } |d t|� }}tjd|�}|�s�t||f��t
|jd��} x|t | �D ]p}|d t|� }}tjd|�}|�s>t||f��|j � \}}t
|�}|dk�rbd}nt|�}| j
||f� �q
W |d t|� }}|d k�s�t||f��| | _|d t|� }}|dk�s�t||f��|d t|� }}tjd|�}|�st||f��t
|jd��}|t| j�k�s&t�|d t|� }}|dk�sNt||f��|d t|� }}tjd |�}|�s~t||f��t
|jd��} | t| j�k�s�t||f��|d t|� }}tjd |�}|�s�t||f��t
|jd��} | | jk�s�t||f��| | _|d t|� }}|d k�s,t||f��y|d t|� }}W n tk
�rX Y nX d�slt||f��dS ) a� Parse the .c file written by pgen. (Internal)
The file looks as follows. The first two lines are always this:
#include "pgenheaders.h"
#include "grammar.h"
After that come four blocks:
1) one or more state definitions
2) a table defining dfas
3) a table defining labels
4) a struct defining the grammar
A state definition has the following form:
- one or more arc arrays, each of the form:
static arc arcs_<n>_<m>[<k>] = {
{<i>, <j>},
...
};
- followed by a state array, of the form:
static state states_<s>[<t>] = {
{<k>, arcs_<n>_<m>},
...
};
zCan't open %s: %sFNr r z#include "pgenheaders.h"
z#include "grammar.h"
z static arc z)static arc arcs_(\d+)_(\d+)\[(\d+)\] = {$z\s+{(\d+), (\d+)},$z};
z'static state states_(\d+)\[(\d+)\] = {$z \s+{(\d+), arcs_(\d+)_(\d+)},$z static dfa dfas\[(\d+)\] = {$z0\s+{(\d+), "(\w+)", (\d+), (\d+), states_(\d+),$� � � � z\s+("(?:\\\d\d\d)*")},$� z!static label labels\[(\d+)\] = {$z\s+{(\d+), (0|"\w+")},$�0z grammar _PyParser_Grammar = {
z
\s+(\d+),$z dfas,
z\s+{(\d+), labels},$z \s+(\d+)$)r
r r �nextr �
startswithr r �list�mapr r �range�append�len�states�groupr r �eval� enumerate�ord�dfas�labels�start�
StopIteration)!r r r r r r Zallarcsr- r �n�m�kZarcs�_�i�j�s�t�stater2 Zndfasr r �x�y�z�firstZ rawbitset�cZbyter3 Znlabelsr4 r r r
r T s�
"
zConverter.parse_graminit_cc C s\ i | _ i | _xJt| j�D ]<\}\}}|tjkrB|dk rB|| j |< q|dkr|| j|< qW dS )z1Create additional useful structures. (Internal).N)�keywords�tokensr0 r3 r �NAME)r Zilabel�type�valuer r r
r � s zConverter.finish_offN)�__name__�
__module__� __qualname__�__doc__r r r r r r r r
r $ s &r )rL r Zpgen2r r ZGrammarr r r r r
�<module> s __pycache__/driver.cpython-36.opt-1.pyc 0000644 00000011730 15046736237 0013653 0 ustar 00 3
\� � @ s� d Z dZddgZddlZddlZddlZddlZddlZddlZddl m
Z
m Z m Z m
Z
mZ G dd� de�Zd d
� Zddd�Zdd� Zdd� Zdd� Zedkr�ejee� �� dS )zZParser driver.
This provides a high-level interface to parse a file into a syntax tree.
z#Guido van Rossum <guido@python.org>�Driver� load_grammar� N� )�grammar�parse�token�tokenize�pgenc @ sH e Zd Zddd�Zddd�Zddd�Zdd d
�Zdd d �Zdd
d�ZdS )r Nc C s& || _ |d krtj� }|| _|| _d S )N)r �logging� getLogger�logger�convert)�selfr r
r � r �,/usr/lib64/python3.6/lib2to3/pgen2/driver.py�__init__ s
zDriver.__init__Fc C sv t j| j| j�}|j� d}d}d } } } } }
d} �x4|D �]} | \}}}} }
|||fkr�|\}
}||
k r�| d|
| 7 } |
}d}||k r�| |
||� 7 } |}|tjtjfkr�| |7 } | \}}|jd�r@|d7 }d}q@|t j
kr�tj | }|�r| j j
dt j| || � |j||| |f��r6|�r4| j j
d� P d} | \}}|jd�r@|d7 }d}q@W t jd||| |f��|jS ) z4Parse a series of tokens and return the syntax tree.r r N� �
z%s %r (prefix=%r)zStop.zincomplete input)r ZParserr r
Zsetupr �COMMENT�NL�endswithr �OPZopmapr �debug�tok_nameZaddtokenZ
ParseErrorZrootnode)r �tokensr �p�lineno�column�type�value�start�endZ line_text�prefixZ quintupleZs_linenoZs_columnr r r � parse_tokens' sR
zDriver.parse_tokensc C s t j|j�}| j||�S )z*Parse a stream and return the syntax tree.)r �generate_tokens�readliner# )r �streamr r r r r �parse_stream_rawW s zDriver.parse_stream_rawc C s | j ||�S )z*Parse a stream and return the syntax tree.)r' )r r& r r r r � parse_stream\ s zDriver.parse_streamc
C s* t j|d|�}z | j||�S |j� X dS )z(Parse a file and return the syntax tree.�rN)�codecs�openr( �close)r �filename�encodingr r& r r r �
parse_file` s zDriver.parse_filec C s t jtj|�j�}| j||�S )z*Parse a string and return the syntax tree.)r r$ �io�StringIOr% r# )r �textr r r r r � parse_stringh s zDriver.parse_string)NN)F)F)F)NF)F) �__name__�
__module__� __qualname__r r# r' r( r/ r3 r r r r r s
0
c C s: t jj| �\}}|dkr d}|| djtttj�� d S )Nz.txtr �.z.pickle)�os�path�splitext�join�map�str�sys� version_info)�gt�head�tailr r r �_generate_pickle_namen s rC � Grammar.txtTFc C s� |dkrt j� }|dkr t| �n|}|s4t|| � r�|jd| � tj| �}|r�|jd|� y|j|� W q� tk
r� } z|jd|� W Y dd}~X q�X nt j
� }|j |� |S )z'Load the grammar (maybe from a pickle).Nz!Generating grammar tables from %sz Writing grammar tables to %szWriting failed: %s) r
r rC �_newer�infor Zgenerate_grammar�dump�OSErrorr �Grammar�load)r@ Zgp�save�forcer �g�er r r r u s
c C s8 t jj| �sdS t jj|�s dS t jj| �t jj|�kS )z0Inquire whether file a was written since file b.FT)r8 r9 �exists�getmtime)�a�br r r rE � s
rE c C sF t jj|�rt|�S tt jj|��}tj| |�}tj � }|j
|� |S )a� Normally, loads a pickled grammar by doing
pkgutil.get_data(package, pickled_grammar)
where *pickled_grammar* is computed from *grammar_source* by adding the
Python version and using a ``.pickle`` extension.
However, if *grammar_source* is an extant file, load_grammar(grammar_source)
is called instead. This facilitates using a packaged grammar file when needed
but preserves load_grammar's automatic regeneration behavior when possible.
) r8 r9 �isfiler rC �basename�pkgutil�get_datar rI �loads)�packageZgrammar_sourceZ pickled_name�datarM r r r �load_packaged_grammar� s
rZ c G sF | st jdd� } tjtjt jdd� x| D ]}t|ddd� q,W dS )z�Main program, when run as a script: produce grammar pickle files.
Calls load_grammar for each argument, a path to a grammar text file.
r Nz %(message)s)�levelr& �formatT)rK rL )r> �argvr
Z basicConfig�INFO�stdoutr )�argsr@ r r r �main� s
ra �__main__)rD NTFN)�__doc__�
__author__�__all__r* r0 r8 r
rU r> r r r r r r �objectr rC r rE rZ ra r4 �exit�intr r r r �<module> s$ P
__pycache__/tokenize.cpython-36.opt-2.pyc 0000644 00000026167 15046736237 0014223 0 ustar 00 3
\NX � = @ s� d Z dZddlZddlZddlmZmZ ddlT ddlm Z dd � e
e �D �d
d d g Z [ ye W n e
k
rz eZ Y nX d
d� Zdd� Zdd� ZdZdZeede � ee� ZdZdZdZdZedd�Zeeeee�Zd Zed d �ee� Z d e Z ee e �Z ed e d! �Z ee e e�Z d"Z!d#Z"d$Z#d%Z$d&Z%ee%d' e%d( �Z&ee%d) e%d* �Z'ed+d,d-d.d/d0d1d2d3� Z(d4Z)ed5d6�Z*ee(e)e*�Z+ee e+e'e�Z,ee, Z-ee%d7 ed8d� e%d9 ed:d� �Z.edee&�Z/eee/e e+e.e� Z0e1e2ej3e-e0e#e$f��\Z4Z5Z6Z7ej3e!�ej3e"�e6e7e6e7e6e7e6e7e6e7e6e7e6e7e6e7e6e7e6e7e6e7e6e7e6e7e6e7e6e7e6e7e6e7e6e7e6e7e6e7e6e7ddddddddd;�4Z8i Z9xd�D ]Z:e:e9e:< �q�W i Z;xd�D ]Z:e:e;e:< �q�W d�Z<G d�d�� d�e=�Z>G d�d�� d�e=�Z?d�d�� Z@e@fd�d
�ZAd�d�� ZBG d�d�� d��ZCej3d�ejD�ZEej3d�ejD�ZFd�d�� ZGd�d�� ZHd�d � ZId�d � ZJeKd�k�r�ddlLZLeMeLjN�dk�rxeAeOeLjNd �jP� n eAeLjQjP� dS )�zKa-Ping Yee <ping@lfw.org>z@GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, Skip Montanaro� N)�BOM_UTF8�lookup)�*� )�tokenc C s g | ]}|d dkr|�qS )r �_� )�.0�xr r �./usr/lib64/python3.6/lib2to3/pgen2/tokenize.py�
<listcomp>% s r �tokenize�generate_tokens�
untokenizec G s ddj | � d S )N�(�|�))�join)�choicesr r r �group0 s r c G s t | � d S )Nr )r )r r r r �any1 s r c G s t | � d S )N�?)r )r r r r �maybe2 s r z[ \f\t]*z #[^\r\n]*z\\\r?\nz [a-zA-Z_]\w*z0[bB]_?[01]+(?:_[01]+)*z(0[xX]_?[\da-fA-F]+(?:_[\da-fA-F]+)*[lL]?z 0[oO]?_?[0-7]+(?:_[0-7]+)*[lL]?z[1-9]\d*(?:_\d+)*[lL]?z0[lL]?z[eE][-+]?\d+(?:_\d+)*z \d+(?:_\d+)*\.(?:\d+(?:_\d+)*)?z\.\d+(?:_\d+)*z \d+(?:_\d+)*z\d+(?:_\d+)*[jJ]z[jJ]z[^'\\]*(?:\\.[^'\\]*)*'z[^"\\]*(?:\\.[^"\\]*)*"z%[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''z%[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""z#(?:[uUrRbBfF]|[rR][bB]|[bBuU][rR])?�'''�"""z '[^\n'\\]*(?:\\.[^\n'\\]*)*'z "[^\n"\\]*(?:\\.[^\n"\\]*)*"z\*\*=?z>>=?z<<=?z<>z!=z//=?z->z[+\-*/%&@|^=<>]=?�~z[][(){}]z\r?\nz[:;.,`@]z'[^\n'\\]*(?:\\.[^\n'\\]*)*�'z"[^\n"\\]*(?:\\.[^\n"\\]*)*�")4r r z'''z"""zr'''zr"""zu'''zu"""zb'''zb"""zf'''zf"""zur'''zur"""zbr'''zbr"""zrb'''zrb"""zR'''zR"""zU'''zU"""zB'''zB"""zF'''zF"""zuR'''zuR"""zUr'''zUr"""zUR'''zUR"""zbR'''zbR"""zBr'''zBr"""zBR'''zBR"""zrB'''zrB"""zRb'''zRb"""zRB'''zRB"""�r�R�u�U�f�F�b�B�r'''�r"""�R'''�R"""�u'''�u"""�U'''�U"""�b'''�b"""�B'''�B"""�f'''�f"""�F'''�F"""�ur'''�ur"""�Ur'''�Ur"""�uR'''�uR"""�UR'''�UR"""�br'''�br"""�Br'''�Br"""�bR'''�bR"""�BR'''�BR"""�rb'''�rb"""�Rb'''�Rb"""�rB'''�rB"""�RB'''�RB"""�r'�r"�R'�R"�u'�u"�U'�U"�b'�b"�B'�B"�f'�f"�F'�F"�ur'�ur"�Ur'�Ur"�uR'�uR"�UR'�UR"�br'�br"�Br'�Br"�bR'�bR"�BR'�BR"�rb'�rb"�Rb'�Rb"�rB'�rB"�RB'�RB"� c @ s e Zd ZdS )�
TokenErrorN)�__name__�
__module__� __qualname__r r r r rw � s rw c @ s e Zd ZdS )�StopTokenizingN)rx ry rz r r r r r{ � s r{ c C s4 |\}}|\}}t d||||t| t|�f � d S )Nz%d,%d-%d,%d: %s %s)�print�tok_name�repr) �typer Zxxx_todo_changemeZxxx_todo_changeme1�lineZsrowZscolZerowZecolr r r �
printtoken� s r� c C s( yt | |� W n tk
r" Y nX d S )N)�
tokenize_loopr{ )�readline�
tokeneaterr r r r
� s
c C s xt | �D ] }||� q
W d S )N)r )r� r� Z
token_infor r r r� � s r� c @ s, e Zd Zdd� Zdd� Zdd� Zdd� Zd S )
� Untokenizerc C s g | _ d| _d| _d S )Nr r )�tokens�prev_row�prev_col)�selfr r r �__init__� s zUntokenizer.__init__c C s* |\}}|| j }|r&| jjd| � d S )N� )r� r� �append)r� �start�row�col�
col_offsetr r r �add_whitespace� s
zUntokenizer.add_whitespacec C s� xv|D ]n}t |�dkr$| j||� P |\}}}}}| j|� | jj|� |\| _| _|ttfkr| jd7 _d| _qW dj | j�S )N� r r � )
�len�compatr� r� r� r� r� �NEWLINE�NLr )r� �iterable�t�tok_typer r� �endr� r r r r � s
zUntokenizer.untokenizec C s� d}g }| j j}|\}}|ttfkr,|d7 }|ttfkr<d}x�|D ]�}|d d� \}}|ttttfkrn|d7 }|tkr�|j|� qBn>|t kr�|j
� qBn*|ttfkr�d}n|r�|r�||d � d}||� qBW d S )NFr� Tr� r ���) r� r� �NAME�NUMBERr� r� �ASYNC�AWAIT�INDENT�DEDENT�pop) r� r r� � startline�indents� toks_append�toknum�tokval�tokr r r r� � s0
zUntokenizer.compatN)rx ry rz r� r� r r� r r r r r� � s r� z&^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)s ^[ \t\f]*(?:[#\r\n]|$)c C sH | d d� j � jdd�}|dks*|jd�r.dS |d ks@|jd
�rDdS | S )N� r �-zutf-8zutf-8-�latin-1�
iso-8859-1� iso-latin-1�latin-1-� iso-8859-1-� iso-latin-1-)r� r� r� )r� r� r� )�lower�replace�
startswith)�orig_enc�encr r r �_get_normal_name s
r� c s� d� d }d}�fdd�}� fdd�}|� }|j t�rHd� |dd � }d }|sT|g fS ||�}|rj||gfS tj|�s~||gfS |� }|s�||gfS ||�}|r�|||gfS |||gfS )
NFzutf-8c s" y� � S t k
r t� S X d S )N)�
StopIteration�bytesr )r� r r � read_or_stop s z%detect_encoding.<locals>.read_or_stopc s� y| j d�}W n tk
r" d S X tj|�}|s6d S t|jd��}y t|�}W n tk
rp td| ��Y nX � r�|j dkr�td��|d7 }|S )N�asciir zunknown encoding: zutf-8zencoding problem: utf-8z-sig)
�decode�UnicodeDecodeError� cookie_re�matchr� r r � LookupError� SyntaxError�name)r� � line_stringr� �encoding�codec)� bom_foundr r � find_cookie& s"
z$detect_encoding.<locals>.find_cookieT� z utf-8-sig)r� r �blank_rer� )r� r� �defaultr� r� �first�secondr )r� r� r �detect_encoding s0
r� c C s t � }|j| �S )N)r� r )r� �utr r r r T s c ! c s� d } }}t jd d }}d \}}d }dg} d }
d} d} d}
�x�y
| � }W n tk
rd d}Y nX |d }dt|� }}|�rF|s�td|��|j|�}|r�|jd� }}t||d |� |||f|| fV d \}}d }nd|�r0|d!d � d k�r0|d"d � d k�r0t|| ||t|�f|fV d}d }qBn|| }|| }qB�nF|dk�rt| �rt|�s`P d}xf||k �r�|| d k�r�|d }n6|| d
k�r�|t d t }n|| dk�r�d}nP |d }�qfW ||k�r�P |
�r�|
V d }
|| dk�r�|| dk�rh||d � j
d�}|t|� }t |||f||t|� f|fV t ||d � ||f|t|�f|fV qBt t f|| dk ||d � ||f|t|�f|fV qB|| d# k�r�| j
|� t|d |� |df||f|fV xt|| d$ k �rJ|| k�rtdd|||f��| d d%� } | �r.| | d& k�r.d} d}
d} td||f||f|fV �q�W | �r�|
�r�| | d' k�r�d} d}
d} n|�s�td|df��d}�x�||k �r8tj||�}|�r |jd�\}}||f||f| }}}|||� || } } | |k�s| dk�r| dk�rt| |||fV �q4| dk�rft} |dk�r8t } n
| �rBd}
|
�rR|
V d }
| | |||fV �q4| dk�r�|
�r�|
V d }
t | |||fV �q4| tk�rt| }|j||�}|�r�|jd�}|||� } |
�r�|
V d }
t| |||f|fV n||f}||d � }|}P �q4| tk�s@| d d� tk�s@| d d
� tk�r�| d( dk�r�||f}t| �pxt| d �pxt| d }||d � d }}|}P n |
�r�|
V d }
t| |||fV �q4| |k�r�| d)k�r�| �r�| dk�r�tnt| |||fV �q�t| |||f} | dk�r |
�r | }
�q�| dk�rx|
�rx|
d tk�rx|
d dk�rxd} | d* } t|
d |
d |
d
|
d fV d }
|
�r�|
V d }
| V nz| d k�r�|
�r�|
V d }
t | |||f|fV d}nF| d k�r�|d }n| d k�r�|d }|
�r�|
V d }
t| |||fV n(t|| ||f||d f|fV |d }�q�W qBW |
�rN|
V d }
x.| dd � D ] } td|df|dfdfV �q\W t d|df|dfdfV d S )+Nr r �
0123456789r� Fr zEOF in multi-line stringr� z\
r� z\
r� � � z#
�#z
z3unindent does not match any outer indentation levelz
<tokenize>zEOF in multi-line statement�.T�
�async�await�def� �\z([{z)]})r� r )r� r ������r� r� r� r� r� r� )r� r� r� ) �stringZ
ascii_lettersr� r� rw r� r� �STRING�
ERRORTOKEN�tabsize�rstrip�COMMENTr� r� r� �IndentationErrorr� �
pseudoprog�spanr� r� �
triple_quoted�endprogs�
single_quotedr� r� r� �OP� ENDMARKER)!r� �lnum�parenlev� continuedZ namechars�numchars�contstr�needcont�contliner� �stashed� async_def�async_def_indent� async_def_nlr� �pos�max�strstart�endprog�endmatchr� �column�
comment_token�nl_pos� pseudomatchr� �spos�eposr �initial�newliner� �indentr r r r i sp
*
�__main__)*r r r& r' r( r) r* r+ r, r- r. r/ r0 r1 r2 r3 r4 r5 r6 r7 r8 r9 r: r; r< r= r> r? r@ rA rB rC rD rE rF rG rH rI rJ rK rL rM )*r r rN rO rP rQ rR rS rT rU rV rW rX rY rZ r[ r\ r] r^ r_ r` ra rb rc rd re rf rg rh ri rj rk rl rm rn ro rp rq rr rs rt ru )R�
__author__� __credits__r� �re�codecsr r Zlib2to3.pgen2.tokenr� r �dir�__all__r� � NameError�strr r r �
Whitespace�Comment�Ignore�Name� Binnumber� Hexnumber� Octnumber� Decnumber� Intnumber�Exponent�
Pointfloat�Expfloat� Floatnumber�
Imagnumber�Number�Single�Double�Single3�Double3Z
_litprefix�Triple�String�Operator�Bracket�Special�Funny�
PlainToken�Token�ContStr� PseudoExtras� PseudoToken�list�map�compileZ tokenprogr� Z single3progZ double3progr� r� r� r� r� � Exceptionrw r{ r� r
r� r� �ASCIIr� r� r� r� r r rx �sysr� �argv�openr� �stdinr r r r �<module> s�
8 I c
__pycache__/driver.cpython-36.pyc 0000644 00000012010 15046736237 0012704 0 ustar 00 3
\� � @ s� d Z dZddgZddlZddlZddlZddlZddlZddlZddl m
Z
m Z m Z m
Z
mZ G dd� de�Zd d
� Zddd�Zdd� Zdd� Zdd� Zedkr�ejee� �� dS )zZParser driver.
This provides a high-level interface to parse a file into a syntax tree.
z#Guido van Rossum <guido@python.org>�Driver� load_grammar� N� )�grammar�parse�token�tokenize�pgenc @ sH e Zd Zddd�Zddd�Zddd�Zdd d
�Zdd d �Zdd
d�ZdS )r Nc C s& || _ |d krtj� }|| _|| _d S )N)r �logging� getLogger�logger�convert)�selfr r
r � r �,/usr/lib64/python3.6/lib2to3/pgen2/driver.py�__init__ s
zDriver.__init__Fc C s� t j| j| j�}|j� d}d}d } } } } }
d} �xR|D �]4} | \}}}} }
|||fkr�||f|ks|t||f|f��|\}
}||
k r�| d|
| 7 } |
}d}||k r�| |
||� 7 } |}|tjtjfkr�| |7 } | \}}|j d�r@|d7 }d}q@|t
j k�rtj | }|�r,| j
jdt
j| || � |j||| |f��rT|�rR| j
jd� P d} | \}}|j d�r@|d7 }d}q@W t jd||| |f��|jS ) z4Parse a series of tokens and return the syntax tree.r r N� �
z%s %r (prefix=%r)zStop.zincomplete input)r ZParserr r
Zsetup�AssertionErrorr �COMMENT�NL�endswithr �OPZopmapr �debug�tok_nameZaddtokenZ
ParseErrorZrootnode)r �tokensr �p�lineno�column�type�value�start�endZ line_text�prefixZ quintupleZs_linenoZs_columnr r r � parse_tokens' sT
zDriver.parse_tokensc C s t j|j�}| j||�S )z*Parse a stream and return the syntax tree.)r �generate_tokens�readliner$ )r �streamr r r r r �parse_stream_rawW s zDriver.parse_stream_rawc C s | j ||�S )z*Parse a stream and return the syntax tree.)r( )r r' r r r r � parse_stream\ s zDriver.parse_streamc
C s* t j|d|�}z | j||�S |j� X dS )z(Parse a file and return the syntax tree.�rN)�codecs�openr) �close)r �filename�encodingr r' r r r �
parse_file` s zDriver.parse_filec C s t jtj|�j�}| j||�S )z*Parse a string and return the syntax tree.)r r% �io�StringIOr& r$ )r �textr r r r r � parse_stringh s zDriver.parse_string)NN)F)F)F)NF)F) �__name__�
__module__� __qualname__r r$ r( r) r0 r4 r r r r r s
0
c C s: t jj| �\}}|dkr d}|| djtttj�� d S )Nz.txtr �.z.pickle)�os�path�splitext�join�map�str�sys� version_info)�gt�head�tailr r r �_generate_pickle_namen s rD � Grammar.txtTFc C s� |dkrt j� }|dkr t| �n|}|s4t|| � r�|jd| � tj| �}|r�|jd|� y|j|� W q� tk
r� } z|jd|� W Y dd}~X q�X nt j
� }|j |� |S )z'Load the grammar (maybe from a pickle).Nz!Generating grammar tables from %sz Writing grammar tables to %szWriting failed: %s) r
r rD �_newer�infor Zgenerate_grammar�dump�OSErrorr �Grammar�load)rA Zgp�save�forcer �g�er r r r u s
c C s8 t jj| �sdS t jj|�s dS t jj| �t jj|�kS )z0Inquire whether file a was written since file b.FT)r9 r: �exists�getmtime)�a�br r r rF � s
rF c C sF t jj|�rt|�S tt jj|��}tj| |�}tj � }|j
|� |S )a� Normally, loads a pickled grammar by doing
pkgutil.get_data(package, pickled_grammar)
where *pickled_grammar* is computed from *grammar_source* by adding the
Python version and using a ``.pickle`` extension.
However, if *grammar_source* is an extant file, load_grammar(grammar_source)
is called instead. This facilitates using a packaged grammar file when needed
but preserves load_grammar's automatic regeneration behavior when possible.
) r9 r: �isfiler rD �basename�pkgutil�get_datar rJ �loads)�packageZgrammar_sourceZ pickled_name�datarN r r r �load_packaged_grammar� s
r[ c G sF | st jdd� } tjtjt jdd� x| D ]}t|ddd� q,W dS )z�Main program, when run as a script: produce grammar pickle files.
Calls load_grammar for each argument, a path to a grammar text file.
r Nz %(message)s)�levelr' �formatT)rL rM )r? �argvr
Z basicConfig�INFO�stdoutr )�argsrA r r r �main� s
rb �__main__)rE NTFN)�__doc__�
__author__�__all__r+ r1 r9 r
rV r? r r r r r r �objectr rD r rF r[ rb r5 �exit�intr r r r �<module> s$ P
__pycache__/driver.cpython-36.opt-2.pyc 0000644 00000007703 15046736237 0013661 0 ustar 00 3
\� � @ s� d Z ddgZddlZddlZddlZddlZddlZddlZddlm Z m
Z
m Z m Z m
Z
G dd� de�Zdd � Zdd
d�Zdd� Zdd� Zdd� Zedkr�ejee� �� dS )z#Guido van Rossum <guido@python.org>�Driver� load_grammar� N� )�grammar�parse�token�tokenize�pgenc @ sH e Zd Zddd�Zddd�Zddd�Zdd d
�Zdd d �Zdd
d�ZdS )r Nc C s& || _ |d krtj� }|| _|| _d S )N)r �logging� getLogger�logger�convert)�selfr r
r � r �,/usr/lib64/python3.6/lib2to3/pgen2/driver.py�__init__ s
zDriver.__init__Fc C sv t j| j| j�}|j� d}d}d } } } } }
d} �x4|D �]} | \}}}} }
|||fkr�|\}
}||
k r�| d|
| 7 } |
}d}||k r�| |
||� 7 } |}|tjtjfkr�| |7 } | \}}|jd�r@|d7 }d}q@|t j
kr�tj | }|�r| j j
dt j| || � |j||| |f��r6|�r4| j j
d� P d} | \}}|jd�r@|d7 }d}q@W t jd||| |f��|jS )Nr r � �
z%s %r (prefix=%r)zStop.zincomplete input)r ZParserr r
Zsetupr �COMMENT�NL�endswithr �OPZopmapr �debug�tok_nameZaddtokenZ
ParseErrorZrootnode)r �tokensr �p�lineno�column�type�value�start�endZ line_text�prefixZ quintupleZs_linenoZs_columnr r r � parse_tokens' sR
zDriver.parse_tokensc C s t j|j�}| j||�S )N)r �generate_tokens�readliner# )r �streamr r r r r �parse_stream_rawW s zDriver.parse_stream_rawc C s | j ||�S )N)r' )r r& r r r r � parse_stream\ s zDriver.parse_streamc
C s* t j|d|�}z | j||�S |j� X d S )N�r)�codecs�openr( �close)r �filename�encodingr r& r r r �
parse_file` s zDriver.parse_filec C s t jtj|�j�}| j||�S )N)r r$ �io�StringIOr% r# )r �textr r r r r � parse_stringh s zDriver.parse_string)NN)F)F)F)NF)F) �__name__�
__module__� __qualname__r r# r' r( r/ r3 r r r r r s
0
c C s: t jj| �\}}|dkr d}|| djtttj�� d S )Nz.txtr �.z.pickle)�os�path�splitext�join�map�str�sys� version_info)�gt�head�tailr r r �_generate_pickle_namen s rC � Grammar.txtTFc C s� |d krt j� }|d kr t| �n|}|s4t|| � r�|jd| � tj| �}|r�|jd|� y|j|� W q� tk
r� } z|jd|� W Y d d }~X q�X nt j
� }|j |� |S )Nz!Generating grammar tables from %sz Writing grammar tables to %szWriting failed: %s) r
r rC �_newer�infor Zgenerate_grammar�dump�OSErrorr �Grammar�load)r@ Zgp�save�forcer �g�er r r r u s
c C s8 t jj| �sdS t jj|�s dS t jj| �t jj|�kS )NFT)r8 r9 �exists�getmtime)�a�br r r rE � s
rE c C sF t jj|�rt|�S tt jj|��}tj| |�}tj � }|j
|� |S )N) r8 r9 �isfiler rC �basename�pkgutil�get_datar rI �loads)�packageZgrammar_sourceZ pickled_name�datarM r r r �load_packaged_grammar� s
rZ c G sF | st jdd � } tjtjt jdd� x| D ]}t|ddd� q,W dS )Nr z %(message)s)�levelr& �formatT)rK rL )r> �argvr
Z basicConfig�INFO�stdoutr )�argsr@ r r r �main� s
ra �__main__)rD NTFN)�
__author__�__all__r* r0 r8 r
rU r> r r r r r r �objectr rC r rE rZ ra r4 �exit�intr r r r �<module> s" P
__pycache__/grammar.cpython-36.opt-1.pyc 0000644 00000015621 15046736237 0014011 0 ustar 00 3
\� � @ sx d Z ddlZddlZddlmZmZ G dd� de�Zdd� Zd Z i Z
x.e j � D ]"Z e rNe j
� \ZZeee�e
e< qNW dS )
a� This module defines the data structures used to represent a grammar.
These are a bit arcane because they are derived from the data
structures used by Python's 'pgen' parser generator.
There's also a table here mapping operators to their names in the
token module; the Python tokenize module reports all operators as the
fallback token code OP, but the parser needs the actual token code.
� N� )�token�tokenizec @ s@ e Zd ZdZdd� Zdd� Zdd� Zdd � Zd
d � Zd d
� Z dS )�Grammara� Pgen parsing tables conversion class.
Once initialized, this class supplies the grammar tables for the
parsing engine implemented by parse.py. The parsing engine
accesses the instance variables directly. The class here does not
provide initialization of the tables; several subclasses exist to
do this (see the conv and pgen modules).
The load() method reads the tables from a pickle file, which is
much faster than the other ways offered by subclasses. The pickle
file is written by calling dump() (after loading the grammar
tables using a subclass). The report() method prints a readable
representation of the tables to stdout, for debugging.
The instance variables are as follows:
symbol2number -- a dict mapping symbol names to numbers. Symbol
numbers are always 256 or higher, to distinguish
them from token numbers, which are between 0 and
255 (inclusive).
number2symbol -- a dict mapping numbers to symbol names;
these two are each other's inverse.
states -- a list of DFAs, where each DFA is a list of
states, each state is a list of arcs, and each
arc is a (i, j) pair where i is a label and j is
a state number. The DFA number is the index into
this list. (This name is slightly confusing.)
Final states are represented by a special arc of
the form (0, j) where j is its own state number.
dfas -- a dict mapping symbol numbers to (DFA, first)
pairs, where DFA is an item from the states list
above, and first is a set of tokens that can
begin this grammar rule (represented by a dict
whose values are always 1).
labels -- a list of (x, y) pairs where x is either a token
number or a symbol number, and y is either None
or a string; the strings are keywords. The label
number is the index in this list; label numbers
are used to mark state transitions (arcs) in the
DFAs.
start -- the number of the grammar's start symbol.
keywords -- a dict mapping keyword strings to arc labels.
tokens -- a dict mapping token numbers to arc labels.
c C s< i | _ i | _g | _i | _dg| _i | _i | _i | _d| _d S )Nr �EMPTY� )r r ) �
symbol2number�
number2symbol�states�dfas�labels�keywords�tokens� symbol2label�start)�self� r �-/usr/lib64/python3.6/lib2to3/pgen2/grammar.py�__init__M s zGrammar.__init__c C s2 t |d�� }t| j�}tj||d� W dQ R X dS )a� Dump the grammar tables to a pickle file.
dump() recursively changes all dict to OrderedDict, so the pickled file
is not exactly the same as what was passed in to dump(). load() uses the
pickled file to create the tables, but only changes OrderedDict to dict
at the top level; it does not recursively change OrderedDict to dict.
So, the loaded tables are different from the original tables that were
passed to load() in that some of the OrderedDict (from the pickled file)
are not changed back to dict. For parsing, this has no effect on
performance because OrderedDict uses dict's __getitem__ with nothing in
between.
�wb� N)�open�_make_deterministic�__dict__�pickle�dump)r �filename�f�dr r r r X s
z Grammar.dumpc C s0 t |d��}tj|�}W dQ R X | jj|� dS )z+Load the grammar tables from a pickle file.�rbN)r r �loadr �update)r r r r r r r r i s z Grammar.loadc C s | j jtj|�� dS )z3Load the grammar tables from a pickle bytes object.N)r r! r �loads)r Zpklr r r r" o s z
Grammar.loadsc C sX | j � }x"dD ]}t||t| |�j� � qW | jdd� |_| jdd� |_| j|_|S ) z#
Copy the grammar.
r r r r
r r N)r r r r
r r )� __class__�setattr�getattr�copyr r
r )r �newZ dict_attrr r r r& s s z Grammar.copyc C sv ddl m } td� || j� td� || j� td� || j� td� || j� td� || j� td| j� d S )
z:Dump the grammar tables to standard output, for debugging.r )�pprintZs2nZn2sr
r r r N)r( �printr r r
r r r )r r( r r r �report� s
zGrammar.reportN)
�__name__�
__module__� __qualname__�__doc__r r r r" r& r* r r r r r s 4
r c C s^ t | t�r&tjtdd� | j� D ���S t | t�r>dd� | D �S t | t�rZtdd� | D ��S | S )Nc s s | ]\}}|t |�fV qd S )N)r )�.0�k�vr r r � <genexpr>� s z&_make_deterministic.<locals>.<genexpr>c S s g | ] }t |��qS r )r )r/ �er r r �
<listcomp>� s z'_make_deterministic.<locals>.<listcomp>c s s | ]}t |�V qd S )N)r )r/ r3 r r r r2 � s )�
isinstance�dict� collections� OrderedDict�sorted�items�list�tuple)�topr r r r � s
r a
( LPAR
) RPAR
[ LSQB
] RSQB
: COLON
, COMMA
; SEMI
+ PLUS
- MINUS
* STAR
/ SLASH
| VBAR
& AMPER
< LESS
> GREATER
= EQUAL
. DOT
% PERCENT
` BACKQUOTE
{ LBRACE
} RBRACE
@ AT
@= ATEQUAL
== EQEQUAL
!= NOTEQUAL
<> NOTEQUAL
<= LESSEQUAL
>= GREATEREQUAL
~ TILDE
^ CIRCUMFLEX
<< LEFTSHIFT
>> RIGHTSHIFT
** DOUBLESTAR
+= PLUSEQUAL
-= MINEQUAL
*= STAREQUAL
/= SLASHEQUAL
%= PERCENTEQUAL
&= AMPEREQUAL
|= VBAREQUAL
^= CIRCUMFLEXEQUAL
<<= LEFTSHIFTEQUAL
>>= RIGHTSHIFTEQUAL
**= DOUBLESTAREQUAL
// DOUBLESLASH
//= DOUBLESLASHEQUAL
-> RARROW
)r. r7 r � r r �objectr r Z opmap_rawZopmap�
splitlines�line�split�op�namer% r r r r �<module>
s y= __pycache__/pgen.cpython-36.opt-2.pyc 0000644 00000022165 15046736237 0013316 0 ustar 00 3
\�5 � @ sd d dl mZmZmZ G dd� dej�ZG dd� de�ZG dd� de�ZG dd � d e�Z dd d �Z
d
S )� )�grammar�token�tokenizec @ s e Zd ZdS )� PgenGrammarN)�__name__�
__module__� __qualname__� r r �*/usr/lib64/python3.6/lib2to3/pgen2/pgen.pyr s r c @ s� e Zd Zd&dd�Zdd� Zdd� Zdd � Zd
d � Zd d
� Zdd� Z dd� Z
dd� Z dd� Z dd� Z
dd� Zdd� Zd d � Zd d � Zd'd d!�Zd"d#� Zd$d%� ZdS )(�ParserGeneratorNc C sl d }|d krt |�}|j}|| _|| _tj|j�| _| j� | j � \| _
| _ |d k rZ|� i | _ | j
� d S )N)�open�close�filename�streamr �generate_tokens�readline� generator�gettoken�parse�dfas� startsymbol�first� addfirstsets)�selfr r Z close_streamr r r
�__init__ s zParserGenerator.__init__c C s* t � }t| jj� �}|j� |j| j� |jd| j� x.|D ]&}dt|j � }||j |< ||j
|< q<W x�|D ]�}| j| }g }xl|D ]d}g }x6t |j j
� �D ]$\} }
|j| j|| �|j|
�f� q�W |jr�|jd|j|�f� |j|� q�W |jj|� || j||�f|j|j | < qlW |j | j |_|S )N� � )r �listr �keys�sort�remover �insert�len�
symbol2numberZ
number2symbol�sorted�arcs�items�append�
make_label�index�isfinal�states�
make_first�start) r �c�names�name�i�dfar+ �stater% �label�nextr r r
� make_grammar s.
z ParserGenerator.make_grammarc C s8 | j | }i }x$t|�D ]}| j||�}d||< qW |S )Nr )r r$ r( )r r. r0 Zrawfirstr r4 �ilabelr r r
r, 4 s
zParserGenerator.make_firstc C s& t |j�}|d j� r�||jkrZ||jkr4|j| S |jj|j| d f� ||j|< |S n>tt|d �}||jkrz|j| S |jj|d f� ||j|< |S n�t |�}|d j� r�||j
kr�|j
| S |jjtj |f� ||j
|< |S n>t j
| }||jk�r|j| S |jj|d f� ||j|< |S d S )Nr )r" Zlabels�isalphar# Z symbol2labelr' �getattrr �tokens�eval�keywords�NAMEr Zopmap)r r. r4 r7 Zitoken�valuer r r
r( = s6
zParserGenerator.make_labelc C s< t | jj� �}|j� x |D ]}|| jkr | j|� q W d S )N)r r r r r � calcfirst)r r/ r0 r r r
r k s
z ParserGenerator.addfirstsetsc C s
| j | }d | j|< |d }i }i }x�|jj� D ]x\}}|| j kr�|| jkrl| j| }|d kr�td| ��n| j|� | j| }|j|� |||< q0d||< |di||< q0W i } xJ|j� D ]>\}}
x4|
D ],} | | kr�td|| || | f ��|| | < q�W q�W || j|< d S )Nr zrecursion for rule %rr zArule %s is ambiguous; %s is in the first sets of %s as well as %s)r r r% r& �
ValueErrorr? �update) r r0 r2 r3 ZtotalsetZ overlapcheckr4 r5 �fsetZinverseZitsfirstZsymbolr r r
r? s s2
zParserGenerator.calcfirstc C s� i }d }x�| j tjkr�x| j tjkr.| j� qW | jtj�}| jtjd� | j� \}}| jtj� | j ||�}t
|�}| j |� t
|�}|||< |d kr
|}q
W ||fS )N�:) �typer � ENDMARKER�NEWLINEr �expectr= �OP� parse_rhs�make_dfar" � simplify_dfa) r r r r0 �a�zr2 ZoldlenZnewlenr r r
r � s"
zParserGenerator.parsec s� � fdd�}� fdd�� t ||�|�g}x�|D ]�}i }x<|jD ]2}x,|jD ]"\}} |d k rJ� | |j|i �� qJW q>W xRt|j� �D ]B\}}
x,|D ]} | j|
kr�P q�W t |
|�} |j| � |j| |� q�W q.W |S )Nc s i }� | |� |S )Nr )r3 �base)�
addclosurer r
�closure� s
z)ParserGenerator.make_dfa.<locals>.closurec s> | |kr d S d|| < x$| j D ]\}}|d kr � ||� q W d S )Nr )r% )r3 rN r4 r5 )rO r r
rO � s z,ParserGenerator.make_dfa.<locals>.addclosure)�DFAState�nfasetr% �
setdefaultr$ r& r' �addarc) r r- �finishrP r+ r3 r% Znfastater4 r5 rR �str )rO r
rJ � s"
zParserGenerator.make_dfac
C s� t d|� |g}x�t|�D ]�\}}t d|||kr4dp6d� x^|jD ]T\}}||kr^|j|�} nt|�} |j|� |d kr�t d| � qBt d|| f � qBW qW d S )NzDump of NFA forz Statez(final)� z -> %dz %s -> %d)�print� enumerater% r) r" r' )
r r0 r- rU Ztodor1 r3 r4 r5 �jr r r
�dump_nfa� s
zParserGenerator.dump_nfac C sl t d|� x\t|�D ]P\}}t d||jr,dp.d� x0t|jj� �D ] \}}t d||j|�f � qBW qW d S )NzDump of DFA forz Statez(final)rW z %s -> %d)rX rY r* r$ r% r& r) )r r0 r2 r1 r3 r4 r5 r r r
�dump_dfa� s
zParserGenerator.dump_dfac C s~ d}xt|rxd}xft |�D ]Z\}}xPt|d t|��D ]:}|| }||kr4||= x|D ]}|j||� qTW d}P q4W qW qW d S )NTFr )rY �ranger" �
unifystate)r r2 Zchangesr1 Zstate_irZ Zstate_jr3 r r r
rK � s
z ParserGenerator.simplify_dfac C s� | j � \}}| jdkr ||fS t� }t� }|j|� |j|� x6| jdkrt| j� | j � \}}|j|� |j|� q@W ||fS d S )N�|)� parse_altr> �NFAStaterT r )r rL rM ZaaZzzr r r
rI � s
zParserGenerator.parse_rhsc C sP | j � \}}x:| jdks*| jtjtjfkrF| j � \}}|j|� |}qW ||fS )N�(�[)rb rc )�
parse_itemr> rD r r= �STRINGrT )r rL �br. �dr r r
r`
s
zParserGenerator.parse_altc C s� | j dkr>| j� | j� \}}| jtjd� |j|� ||fS | j� \}}| j }|dkr`||fS | j� |j|� |dkr�||fS ||fS d S )Nrc �]�+�*)ri rj )r> r rI rG r rH rT �
parse_atom)r rL rM r> r r r
rd s
zParserGenerator.parse_itemc C s� | j dkr4| j� | j� \}}| jtjd� ||fS | jtjtjfkrpt � }t � }|j
|| j � | j� ||fS | j d| j| j � d S )Nrb �)z+expected (...) or NAME or STRING, got %s/%s) r> r rI rG r rH rD r= re ra rT � raise_error)r rL rM r r r
rk ( s
zParserGenerator.parse_atomc C sD | j |ks |d k r2| j|kr2| jd||| j | j� | j}| j� |S )Nzexpected %s/%s, got %s/%s)rD r> rm r )r rD r> r r r
rG 9 s zParserGenerator.expectc C sJ t | j�}x"|d tjtjfkr,t | j�}q W |\| _| _| _| _| _ d S )Nr )
r5 r r �COMMENT�NLrD r> Zbegin�end�line)r �tupr r r
r A s
zParserGenerator.gettokenc G s^ |r8y || }W n& dj |gttt|�� �}Y nX t|| j| jd | jd | jf��d S )N� r r )�joinr �map�str� SyntaxErrorr rp rq )r �msg�argsr r r
rm H s zParserGenerator.raise_error)N)N)r r r r r6 r, r( r r? r rJ r[ r\ rK rI r` rd rk rG r rm r r r r
r
s$
. $
r c @ s e Zd Zdd� Zddd�ZdS )ra c C s
g | _ d S )N)r% )r r r r
r S s zNFAState.__init__Nc C s | j j||f� d S )N)r% r' )r r5 r4 r r r
rT V s zNFAState.addarc)N)r r r r rT r r r r
ra Q s ra c @ s0 e Zd Zdd� Zdd� Zdd� Zdd� Zd Zd S )
rQ c C s || _ ||k| _i | _d S )N)rR r* r% )r rR �finalr r r
r ] s
zDFAState.__init__c C s || j |< d S )N)r% )r r5 r4 r r r
rT e s zDFAState.addarcc C s. x(| j j� D ]\}}||kr || j |< q W d S )N)r% r& )r �old�newr4 r5 r r r
r^ k s zDFAState.unifystatec C sX | j |j krdS t| j�t|j�kr(dS x*| jj� D ] \}}||jj|�k r4dS q4W dS )NFT)r* r" r% r& �get)r �otherr4 r5 r r r
�__eq__p s zDFAState.__eq__N)r r r r rT r^ r �__hash__r r r r
rQ [ s
rQ � Grammar.txtc C s t | �}|j� S )N)r r6 )r �pr r r
�generate_grammar� s r� N)r� ) rW r r r ZGrammarr �objectr ra rQ r� r r r r
�<module> s I
% __pycache__/token.cpython-36.pyc 0000644 00000003474 15046736237 0012547 0 ustar 00 3
\ � @ sP d Z dZdZdZdZdZdZdZdZd Z d
Z
d Z d Z d
Z
dZdZdZdZdZdZdZdZdZdZdZdZdZdZd Z d Z d Z d Z d Z d!Z!d"Z"d#Z#d$Z$d%Z%d&Z&d'Z'd(Z(d)Z)d*Z*d+Z+d,Z,d-Z-d.Z.d/Z/d0Z0d1Z1d2Z2d3Z3d4Z4d5Z5d6Z6d7Z7d8Z8d9Z9d:Z:d;Z;d<Z<d=Z=i Z>x6e?e@� jA� �D ]$\ZBZCeDeC�eDd�k�r eBe>eC< �q W d>d?� ZEd@dA� ZFdBdC� ZGdDS )Ez!Token constants (from "token.h").� � � � � � � � � � �
� � �
� � � � � � � � � � � � � � � � � � � �! �" �# �$ �% �& �' �( �) �* �+ �, �- �. �/ �0 �1 �2 �3 �4 �5 �6 �7 �8 �9 �: �; � c C s | t k S )N)� NT_OFFSET)�x� r@ �+/usr/lib64/python3.6/lib2to3/pgen2/token.py�
ISTERMINALN s rB c C s | t kS )N)r> )r? r@ r@ rA �
ISNONTERMINALQ s rC c C s | t kS )N)� ENDMARKER)r? r@ r@ rA �ISEOFT s rE N)H�__doc__rD �NAME�NUMBER�STRING�NEWLINE�INDENT�DEDENT�LPAR�RPAR�LSQB�RSQB�COLON�COMMA�SEMI�PLUS�MINUS�STAR�SLASH�VBAR�AMPER�LESS�GREATER�EQUAL�DOT�PERCENTZ BACKQUOTE�LBRACE�RBRACE�EQEQUAL�NOTEQUAL� LESSEQUAL� GREATEREQUAL�TILDE�
CIRCUMFLEX� LEFTSHIFT�
RIGHTSHIFT�
DOUBLESTAR� PLUSEQUAL�MINEQUAL� STAREQUAL�
SLASHEQUAL� PERCENTEQUAL�
AMPEREQUAL� VBAREQUAL�CIRCUMFLEXEQUAL�LEFTSHIFTEQUAL�RIGHTSHIFTEQUAL�DOUBLESTAREQUAL� DOUBLESLASH�DOUBLESLASHEQUAL�AT�ATEQUAL�OP�COMMENT�NL�RARROW�AWAIT�ASYNC�
ERRORTOKEN�N_TOKENSr> �tok_name�list�globals�items�_nameZ_value�typerB rC rE r@ r@ r@ rA �<module> s� __pycache__/parse.cpython-36.opt-1.pyc 0000644 00000014233 15046736237 0013473 0 ustar 00 3
\u � @ s4 d Z ddlmZ G dd� de�ZG dd� de�ZdS )z�Parser engine for the grammar tables generated by pgen.
The grammar table must be loaded first.
See Parser/parser.c in the Python distribution for additional info on
how this parsing engine works.
� )�tokenc @ s e Zd ZdZdd� ZdS )�
ParseErrorz(Exception to signal the parser is stuck.c C s4 t j| d||||f � || _|| _|| _|| _d S )Nz!%s: type=%r, value=%r, context=%r)� Exception�__init__�msg�type�value�context)�selfr r r r � r �+/usr/lib64/python3.6/lib2to3/pgen2/parse.pyr s zParseError.__init__N)�__name__�
__module__� __qualname__�__doc__r r r r r r s r c @ sL e Zd ZdZddd�Zddd�Zdd� Zd d
� Zd d � Zd
d� Z dd� Z
dS )�Parsera5 Parser engine.
The proper usage sequence is:
p = Parser(grammar, [converter]) # create instance
p.setup([start]) # prepare for parsing
<for each input token>:
if p.addtoken(...): # parse a token; may raise ParseError
break
root = p.rootnode # root of abstract syntax tree
A Parser instance may be reused by calling setup() repeatedly.
A Parser instance contains state pertaining to the current token
sequence, and should not be used concurrently by different threads
to parse separate token sequences.
See driver.py for how to get input tokens by tokenizing a file or
string.
Parsing is complete when addtoken() returns True; the root of the
abstract syntax tree can then be retrieved from the rootnode
instance variable. When a syntax error occurs, addtoken() raises
the ParseError exception. There is no error recovery; the parser
cannot be used after a syntax error was reported (but it can be
reinitialized by calling setup()).
Nc C s || _ |pdd� | _dS )a� Constructor.
The grammar argument is a grammar.Grammar instance; see the
grammar module for more information.
The parser is not ready yet for parsing; you must call the
setup() method to get it started.
The optional convert argument is a function mapping concrete
syntax tree nodes to abstract syntax tree nodes. If not
given, no conversion is done and the syntax tree produced is
the concrete syntax tree. If given, it must be a function of
two arguments, the first being the grammar (a grammar.Grammar
instance), and the second being the concrete syntax tree node
to be converted. The syntax tree is converted from the bottom
up.
A concrete syntax tree node is a (type, value, context, nodes)
tuple, where type is the node type (a token or symbol number),
value is None for symbols and a string for tokens, context is
None or an opaque value used for error reporting (typically a
(lineno, offset) pair), and nodes is a list of children for
symbols, and None for tokens.
An abstract syntax tree node may be anything; this is entirely
up to the converter function.
c S s |S )Nr )�grammar�noder r r �<lambda>W s z!Parser.__init__.<locals>.<lambda>N)r �convert)r
r r r r r r 9 s zParser.__init__c C sH |dkr| j j}|ddg f}| j j| d|f}|g| _d| _t� | _dS )a� Prepare for parsing.
This *must* be called before starting to parse.
The optional argument is an alternative start symbol; it
defaults to the grammar's start symbol.
You can use a Parser instance to parse any number of programs;
each time you call setup() the parser is reset to an initial
state determined by the (implicit or explicit) start symbol.
N� )r �start�dfas�stack�rootnode�set�
used_names)r
r �newnodeZ
stackentryr r r �setupY s
z Parser.setupc C s: | j |||�}�x$| jd \}}}|\}} || }
� x�|
D ]�\} } | jj| \}
}|| kr�| j||| |� | }x@|| d|fgkr�| j� | js�dS | jd
\}}}|\}} qpW dS |
dkr:| jj|
}|\}}||kr:| j|
| jj|
| |� P q:W d|f|
k�r$| j� | j�s2td|||��qtd|||��qW dS ) z<Add a token; return True iff this is the end of the program.r r TF� ztoo much inputz bad inputN���r ) �classifyr r Zlabels�shift�popr �pushr )r
r r r �ilabel�dfa�stater Zstates�firstZarcs�i�newstate�t�vZitsdfaZ itsstatesZitsfirstr r r �addtokenq s: zParser.addtokenc C sX |t jkr0| jj|� | jjj|�}|dk r0|S | jjj|�}|dkrTtd|||��|S )z&Turn a token into a label. (Internal)Nz bad token) r �NAMEr �addr �keywords�get�tokensr )r
r r r r% r r r r! � s
zParser.classifyc C sT | j d \}}}|||df}| j| j|�}|dk r@|d j|� |||f| j d< dS )zShift a token. (Internal)r Nr r r )r r r �append) r
r r r* r r& r' r r r r r r" � s z Parser.shiftc C sB | j d \}}}|d|g f}|||f| j d< | j j|d|f� dS )z Push a nonterminal. (Internal)r Nr r r )r r3 ) r
r Znewdfar* r r& r' r r r r r r$ � s z Parser.pushc C s` | j j� \}}}| j| j|�}|dk r\| j rL| j d \}}}|d j|� n|| _| j| j_dS )z Pop a nonterminal. (Internal)Nr r r )r r# r r r3 r r )r
ZpopdfaZpopstateZpopnoder r& r' r r r r r# � s z
Parser.pop)N)N) r
r r r r r r- r! r" r$ r# r r r r r s
0 r N)r � r r r �objectr r r r r �<module> s __pycache__/parse.cpython-36.opt-2.pyc 0000644 00000006006 15046736237 0013473 0 ustar 00 3
\u � @ s0 d dl mZ G dd� de�ZG dd� de�ZdS )� )�tokenc @ s e Zd Zdd� ZdS )�
ParseErrorc C s4 t j| d||||f � || _|| _|| _|| _d S )Nz!%s: type=%r, value=%r, context=%r)� Exception�__init__�msg�type�value�context)�selfr r r r � r �+/usr/lib64/python3.6/lib2to3/pgen2/parse.pyr s zParseError.__init__N)�__name__�
__module__� __qualname__r r r r r r s r c @ sH e Zd Zddd�Zddd�Zdd� Zdd � Zd
d � Zd d
� Zdd� Z dS )�ParserNc C s || _ |pdd� | _d S )Nc S s |S )Nr )�grammar�noder r r �<lambda>W s z!Parser.__init__.<locals>.<lambda>)r �convert)r
r r r r r r 9 s zParser.__init__c C sH |d kr| j j}|d d g f}| j j| d|f}|g| _d | _t� | _d S )N� )r �start�dfas�stack�rootnode�set�
used_names)r
r �newnodeZ
stackentryr r r �setupY s
z Parser.setupc C s: | j |||�}�x$| jd \}}}|\}} || }
� x�|
D ]�\} } | jj| \}
}|| kr�| j||| |� | }x@|| d|fgkr�| j� | js�dS | jd \}}}|\}} qpW dS |
dkr:| jj|
}|\}}||kr:| j|
| jj|
| |� P q:W d|f|
k�r$| j� | j�s2td|||��qtd|||��qW d S )
Nr r TF� ztoo much inputz bad input���r ) �classifyr r Zlabels�shift�popr �pushr )r
r r r �ilabel�dfa�stater Zstates�firstZarcs�i�newstate�t�vZitsdfaZ itsstatesZitsfirstr r r �addtokenq s: zParser.addtokenc C sX |t jkr0| jj|� | jjj|�}|d k r0|S | jjj|�}|d krTtd|||��|S )Nz bad token) r �NAMEr �addr �keywords�get�tokensr )r
r r r r$ r r r r � s
zParser.classifyc C sT | j d \}}}|||d f}| j| j|�}|d k r@|d j|� |||f| j d< d S )Nr r r r )r r r �append) r
r r r) r r% r& r r r r r r! � s z Parser.shiftc C sB | j d \}}}|d |g f}|||f| j d< | j j|d|f� d S )Nr r r r )r r2 ) r
r Znewdfar) r r% r& r r r r r r# � s z Parser.pushc C s` | j j� \}}}| j| j|�}|d k r\| j rL| j d \}}}|d j|� n|| _| j| j_d S )Nr r r )r r" r r r2 r r )r
ZpopdfaZpopstateZpopnoder r% r& r r r r r" � s z
Parser.pop)N)N)
r
r r r r r, r r! r# r" r r r r r s
0 r N)� r r r �objectr r r r r �<module> s __pycache__/token.cpython-36.opt-2.pyc 0000644 00000003412 15046736237 0013477 0 ustar 00 3
\ � @ sL d Z dZdZdZdZdZdZdZdZd Z d
Z
d Z d Z d
Z
dZdZdZdZdZdZdZdZdZdZdZdZdZdZd Z d Z d Z d Z d Z d!Z!d"Z"d#Z#d$Z$d%Z%d&Z&d'Z'd(Z(d)Z)d*Z*d+Z+d,Z,d-Z-d.Z.d/Z/d0Z0d1Z1d2Z2d3Z3d4Z4d5Z5d6Z6d7Z7d8Z8d9Z9d:Z:d;Z;d<Z<i Z=x6e>e?� j@� �D ]$\ZAZBeCeB�eCd �k�reAe=eB< �qW d=d>� ZDd?d@� ZEdAdB� ZFdCS )D� � � � � � � � � � �
� � �
� � � � � � � � � � � � � � � � � � � �! �" �# �$ �% �& �' �( �) �* �+ �, �- �. �/ �0 �1 �2 �3 �4 �5 �6 �7 �8 �9 �: �; � c C s | t k S )N)� NT_OFFSET)�x� r@ �+/usr/lib64/python3.6/lib2to3/pgen2/token.py�
ISTERMINALN s rB c C s | t kS )N)r> )r? r@ r@ rA �
ISNONTERMINALQ s rC c C s | t kS )N)� ENDMARKER)r? r@ r@ rA �ISEOFT s rE N)GrD �NAME�NUMBER�STRING�NEWLINE�INDENT�DEDENT�LPAR�RPAR�LSQB�RSQB�COLON�COMMA�SEMI�PLUS�MINUS�STAR�SLASH�VBAR�AMPER�LESS�GREATER�EQUAL�DOT�PERCENTZ BACKQUOTE�LBRACE�RBRACE�EQEQUAL�NOTEQUAL� LESSEQUAL� GREATEREQUAL�TILDE�
CIRCUMFLEX� LEFTSHIFT�
RIGHTSHIFT�
DOUBLESTAR� PLUSEQUAL�MINEQUAL� STAREQUAL�
SLASHEQUAL� PERCENTEQUAL�
AMPEREQUAL� VBAREQUAL�CIRCUMFLEXEQUAL�LEFTSHIFTEQUAL�RIGHTSHIFTEQUAL�DOUBLESTAREQUAL� DOUBLESLASH�DOUBLESLASHEQUAL�AT�ATEQUAL�OP�COMMENT�NL�RARROW�AWAIT�ASYNC�
ERRORTOKEN�N_TOKENSr> �tok_name�list�globals�items�_nameZ_value�typerB rC rE r@ r@ r@ rA �<module> s� __pycache__/grammar.cpython-36.opt-2.pyc 0000644 00000006253 15046736237 0014013 0 ustar 00 3
\� � @ st d dl Z d dlZddlmZmZ G dd� de�Zdd� ZdZi Z x.ej
� D ]"Z e rJe j � \Z
Zeee�e e
< qJW dS ) � N� )�token�tokenizec @ s< e Zd Zdd� Zdd� Zdd� Zdd� Zd d
� Zd d � Zd
S )�Grammarc C s< i | _ i | _g | _i | _dg| _i | _i | _i | _d| _d S )Nr �EMPTY� )r r ) �
symbol2number�
number2symbol�states�dfas�labels�keywords�tokens� symbol2label�start)�self� r �-/usr/lib64/python3.6/lib2to3/pgen2/grammar.py�__init__M s zGrammar.__init__c C s2 t |d�� }t| j�}tj||d� W d Q R X d S )N�wb� )�open�_make_deterministic�__dict__�pickle�dump)r �filename�f�dr r r r X s
z Grammar.dumpc C s0 t |d��}tj|�}W d Q R X | jj|� d S )N�rb)r r �loadr �update)r r r r r r r r i s z Grammar.loadc C s | j jtj|�� d S )N)r r! r �loads)r Zpklr r r r" o s z
Grammar.loadsc C sX | j � }x"dD ]}t||t| |�j� � qW | jd d � |_| jd d � |_| j|_|S )Nr r r r
r r )r r r r
r r )� __class__�setattr�getattr�copyr r
r )r �newZ dict_attrr r r r& s s z Grammar.copyc C sv ddl m } td� || j� td� || j� td� || j� td� || j� td� || j� td| j� d S ) Nr )�pprintZs2nZn2sr
r r r )r( �printr r r
r r r )r r( r r r �report� s
zGrammar.reportN) �__name__�
__module__� __qualname__r r r r" r& r* r r r r r s 6
r c C s^ t | t�r&tjtdd� | j� D ���S t | t�r>dd� | D �S t | t�rZtdd� | D ��S | S )Nc s s | ]\}}|t |�fV qd S )N)r )�.0�k�vr r r � <genexpr>� s z&_make_deterministic.<locals>.<genexpr>c S s g | ] }t |��qS r )r )r. �er r r �
<listcomp>� s z'_make_deterministic.<locals>.<listcomp>c s s | ]}t |�V qd S )N)r )r. r2 r r r r1 � s )�
isinstance�dict� collections� OrderedDict�sorted�items�list�tuple)�topr r r r � s
r a
( LPAR
) RPAR
[ LSQB
] RSQB
: COLON
, COMMA
; SEMI
+ PLUS
- MINUS
* STAR
/ SLASH
| VBAR
& AMPER
< LESS
> GREATER
= EQUAL
. DOT
% PERCENT
` BACKQUOTE
{ LBRACE
} RBRACE
@ AT
@= ATEQUAL
== EQEQUAL
!= NOTEQUAL
<> NOTEQUAL
<= LESSEQUAL
>= GREATEREQUAL
~ TILDE
^ CIRCUMFLEX
<< LEFTSHIFT
>> RIGHTSHIFT
** DOUBLESTAR
+= PLUSEQUAL
-= MINEQUAL
*= STAREQUAL
/= SLASHEQUAL
%= PERCENTEQUAL
&= AMPEREQUAL
|= VBAREQUAL
^= CIRCUMFLEXEQUAL
<<= LEFTSHIFTEQUAL
>>= RIGHTSHIFTEQUAL
**= DOUBLESTAREQUAL
// DOUBLESLASH
//= DOUBLESLASHEQUAL
-> RARROW
)r6 r � r r �objectr r Z opmap_rawZopmap�
splitlines�line�split�op�namer% r r r r �<module> s y= __pycache__/conv.cpython-36.opt-2.pyc 0000644 00000007120 15046736237 0013324 0 ustar 00 3
\�% � @ s. d dl Z d dlmZmZ G dd� dej�ZdS )� N)�grammar�tokenc @ s, e Zd Zdd� Zdd� Zdd� Zdd� Zd S )
� Converterc C s | j |� | j|� | j� d S )N)�parse_graminit_h�parse_graminit_c�
finish_off)�selfZ
graminit_hZ
graminit_c� r �*/usr/lib64/python3.6/lib2to3/pgen2/conv.py�run/ s
z
Converter.runc C s� y t |�}W n0 tk
r< } ztd||f � dS d }~X nX i | _i | _d}xn|D ]f}|d7 }tjd|�}| r�|j� r�td|||j� f � qT|j� \}}t |�}|| j|< || j|< qTW dS )NzCan't open %s: %sFr � z^#define\s+(\w+)\s+(\d+)$z%s(%s): can't parse %sT)
�open�OSError�printZ
symbol2numberZ
number2symbol�re�match�strip�groups�int) r �filename�f�err�lineno�line�mo�symbol�numberr r r
r 5 s&
zConverter.parse_graminit_hc ! C s� y t |�}W n0 tk
r< } ztd||f � dS d }~X nX d}|d t|� }}|d t|� }}|d t|� }}i }g }�x�|jd��rx�|jd��rLtjd|�}ttt |j
� ��\} }
} g } xRt | �D ]F}
|d t|� }}tjd|�}ttt |j
� ��\}}| j ||f� q�W |d t|� }}| || |
f< |d t|� }}q�W tjd|�}ttt |j
� ��\}}g }x^t |�D ]R}
|d t|� }}tjd |�}ttt |j
� ��\} } }
|| |
f } |j | � �q~W |j |� |d t|� }}|d t|� }}q�W || _
i }tjd
|�}t |jd��}x�t |�D ]�}|d t|� }}tjd |�}|jd �}ttt |jdd
dd���\}}}}|| }|d t|� }}tjd|�}i }t|jd��}xPt|�D ]D\}} t| �} x0t d�D ]$}| d|> @ �r�d||d | < �q�W �q�W ||f||< �q4W |d t|� }}|| _g } |d t|� }}tjd|�}t |jd��} xjt | �D ]^}|d t|� }}tjd|�}|j
� \}}t |�}|dk�r�d }nt|�}| j ||f� �qpW |d t|� }}| | _|d t|� }}|d t|� }}tjd|�}t |jd��}|d t|� }}|d t|� }}tjd|�}t |jd��} |d t|� }}tjd|�}t |jd��} | | _|d t|� }}y|d t|� }}W n tk
�r� Y nX d S )NzCan't open %s: %sFr r z static arc z)static arc arcs_(\d+)_(\d+)\[(\d+)\] = {$z\s+{(\d+), (\d+)},$z'static state states_(\d+)\[(\d+)\] = {$z \s+{(\d+), arcs_(\d+)_(\d+)},$z static dfa dfas\[(\d+)\] = {$z0\s+{(\d+), "(\w+)", (\d+), (\d+), states_(\d+),$� � � � z\s+("(?:\\\d\d\d)*")},$� z!static label labels\[(\d+)\] = {$z\s+{(\d+), (0|"\w+")},$�0z
\s+(\d+),$z\s+{(\d+), labels},$z \s+(\d+)$)r
r r �next�
startswithr r �list�mapr r �range�append�states�group�eval� enumerate�ord�dfas�labels�start�
StopIteration)!r r r r r r Zallarcsr) r �n�m�kZarcs�_�i�j�s�t�stater. Zndfasr r �x�y�z�firstZ rawbitset�cZbyter/ Znlabelsr0 r r r
r T s�
"
zConverter.parse_graminit_cc C s\ i | _ i | _xJt| j�D ]<\}\}}|tjkrB|d k rB|| j |< q|d kr|| j|< qW d S )N)�keywords�tokensr, r/ r �NAME)r Zilabel�type�valuer r r
r � s zConverter.finish_offN)�__name__�
__module__� __qualname__r r r r r r r r
r $ s
&r )r Zpgen2r r ZGrammarr r r r r
�<module> s __pycache__/parse.cpython-36.pyc 0000644 00000014271 15046736237 0012536 0 ustar 00 3
\u � @ s4 d Z ddlmZ G dd� de�ZG dd� de�ZdS )z�Parser engine for the grammar tables generated by pgen.
The grammar table must be loaded first.
See Parser/parser.c in the Python distribution for additional info on
how this parsing engine works.
� )�tokenc @ s e Zd ZdZdd� ZdS )�
ParseErrorz(Exception to signal the parser is stuck.c C s4 t j| d||||f � || _|| _|| _|| _d S )Nz!%s: type=%r, value=%r, context=%r)� Exception�__init__�msg�type�value�context)�selfr r r r � r �+/usr/lib64/python3.6/lib2to3/pgen2/parse.pyr s zParseError.__init__N)�__name__�
__module__� __qualname__�__doc__r r r r r r s r c @ sL e Zd ZdZddd�Zddd�Zdd� Zd d
� Zd d � Zd
d� Z dd� Z
dS )�Parsera5 Parser engine.
The proper usage sequence is:
p = Parser(grammar, [converter]) # create instance
p.setup([start]) # prepare for parsing
<for each input token>:
if p.addtoken(...): # parse a token; may raise ParseError
break
root = p.rootnode # root of abstract syntax tree
A Parser instance may be reused by calling setup() repeatedly.
A Parser instance contains state pertaining to the current token
sequence, and should not be used concurrently by different threads
to parse separate token sequences.
See driver.py for how to get input tokens by tokenizing a file or
string.
Parsing is complete when addtoken() returns True; the root of the
abstract syntax tree can then be retrieved from the rootnode
instance variable. When a syntax error occurs, addtoken() raises
the ParseError exception. There is no error recovery; the parser
cannot be used after a syntax error was reported (but it can be
reinitialized by calling setup()).
Nc C s || _ |pdd� | _dS )a� Constructor.
The grammar argument is a grammar.Grammar instance; see the
grammar module for more information.
The parser is not ready yet for parsing; you must call the
setup() method to get it started.
The optional convert argument is a function mapping concrete
syntax tree nodes to abstract syntax tree nodes. If not
given, no conversion is done and the syntax tree produced is
the concrete syntax tree. If given, it must be a function of
two arguments, the first being the grammar (a grammar.Grammar
instance), and the second being the concrete syntax tree node
to be converted. The syntax tree is converted from the bottom
up.
A concrete syntax tree node is a (type, value, context, nodes)
tuple, where type is the node type (a token or symbol number),
value is None for symbols and a string for tokens, context is
None or an opaque value used for error reporting (typically a
(lineno, offset) pair), and nodes is a list of children for
symbols, and None for tokens.
An abstract syntax tree node may be anything; this is entirely
up to the converter function.
c S s |S )Nr )�grammar�noder r r �<lambda>W s z!Parser.__init__.<locals>.<lambda>N)r �convert)r
r r r r r r 9 s zParser.__init__c C sH |dkr| j j}|ddg f}| j j| d|f}|g| _d| _t� | _dS )a� Prepare for parsing.
This *must* be called before starting to parse.
The optional argument is an alternative start symbol; it
defaults to the grammar's start symbol.
You can use a Parser instance to parse any number of programs;
each time you call setup() the parser is reset to an initial
state determined by the (implicit or explicit) start symbol.
N� )r �start�dfas�stack�rootnode�set�
used_names)r
r �newnodeZ
stackentryr r r �setupY s
z Parser.setupc C sF | j |||�}�x0| jd \}}}|\}} || }
�x|
D ]�\} } | jj| \}
}|| kr�|
dk sft�| j||| |� | }x@|| d|fgkr�| j� | js�dS | jd
\}}}|\}} q|W dS |
dkr:| jj|
}|\}}||kr:| j|
| jj|
| |� P q:W d|f|
k�r0| j� | j�s>t d|||��qt d|||��qW dS ) z<Add a token; return True iff this is the end of the program.r � r TFztoo much inputz bad inputN���r )
�classifyr r Zlabels�AssertionError�shift�popr �pushr )r
r r r �ilabel�dfa�stater Zstates�firstZarcs�i�newstate�t�vZitsdfaZ itsstatesZitsfirstr r r �addtokenq s< zParser.addtokenc C sX |t jkr0| jj|� | jjj|�}|dk r0|S | jjj|�}|dkrTtd|||��|S )z&Turn a token into a label. (Internal)Nz bad token) r �NAMEr �addr �keywords�get�tokensr )r
r r r r& r r r r! � s
zParser.classifyc C sT | j d \}}}|||df}| j| j|�}|dk r@|d j|� |||f| j d< dS )zShift a token. (Internal)r Nr r r )r r r �append) r
r r r+ r r' r( r r r r r r# � s z Parser.shiftc C sB | j d \}}}|d|g f}|||f| j d< | j j|d|f� dS )z Push a nonterminal. (Internal)r Nr r r )r r4 ) r
r Znewdfar+ r r' r( r r r r r r% � s z Parser.pushc C s` | j j� \}}}| j| j|�}|dk r\| j rL| j d \}}}|d j|� n|| _| j| j_dS )z Pop a nonterminal. (Internal)Nr r r )r r$ r r r4 r r )r
ZpopdfaZpopstateZpopnoder r' r( r r r r r$ � s z
Parser.pop)N)N) r
r r r r r r. r! r# r% r$ r r r r r s
0 r N)r � r r r �objectr r r r r �<module> s __pycache__/__init__.cpython-36.pyc 0000644 00000000237 15046736237 0013160 0 ustar 00 3
\� � @ s d Z dS )zThe pgen2 package.N)�__doc__� r r �./usr/lib64/python3.6/lib2to3/pgen2/__init__.py�<module> s __pycache__/__init__.cpython-36.opt-1.pyc 0000644 00000000237 15046736237 0014117 0 ustar 00 3
\� � @ s d Z dS )zThe pgen2 package.N)�__doc__� r r �./usr/lib64/python3.6/lib2to3/pgen2/__init__.py�<module> s __pycache__/conv.cpython-36.opt-1.pyc 0000644 00000014004 15046736237 0013322 0 ustar 00 3
\�% � @ s2 d Z ddlZddlmZmZ G dd� dej�ZdS )a� Convert graminit.[ch] spit out by pgen to Python code.
Pgen is the Python parser generator. It is useful to quickly create a
parser from a grammar file in Python's grammar notation. But I don't
want my parsers to be written in C (yet), so I'm translating the
parsing tables to Python data structures and writing a Python parse
engine.
Note that the token numbers are constants determined by the standard
Python tokenizer. The standard token module defines these numbers and
their names (the names are not used much). The token numbers are
hardcoded into the Python tokenizer and into pgen. A Python
implementation of the Python tokenizer is also available, in the
standard tokenize module.
On the other hand, symbol numbers (representing the grammar's
non-terminals) are assigned by pgen based on the actual grammar
input.
Note: this module is pretty much obsolete; the pgen module generates
equivalent grammar tables directly from the Grammar.txt input file
without having to invoke the Python pgen C program.
� N)�grammar�tokenc @ s0 e Zd ZdZdd� Zdd� Zdd� Zdd � Zd
S ) � Convertera2 Grammar subclass that reads classic pgen output files.
The run() method reads the tables as produced by the pgen parser
generator, typically contained in two C files, graminit.h and
graminit.c. The other methods are for internal use only.
See the base class for more documentation.
c C s | j |� | j|� | j� dS )z<Load the grammar tables from the text files written by pgen.N)�parse_graminit_h�parse_graminit_c�
finish_off)�selfZ
graminit_hZ
graminit_c� r �*/usr/lib64/python3.6/lib2to3/pgen2/conv.py�run/ s
z
Converter.runc C s� y t |�}W n0 tk
r< } ztd||f � dS d}~X nX i | _i | _d}xn|D ]f}|d7 }tjd|�}| r�|j� r�td|||j� f � qT|j� \}}t |�}|| j|< || j|< qTW dS ) z�Parse the .h file written by pgen. (Internal)
This file is a sequence of #define statements defining the
nonterminals of the grammar as numbers. We build two tables
mapping the numbers to names and back.
zCan't open %s: %sFNr � z^#define\s+(\w+)\s+(\d+)$z%s(%s): can't parse %sT)
�open�OSError�printZ
symbol2numberZ
number2symbol�re�match�strip�groups�int) r �filename�f�err�lineno�line�mo�symbol�numberr r r
r 5 s&
zConverter.parse_graminit_hc ! C s� y t |�}W n0 tk
r< } ztd||f � dS d}~X nX d}|d t|� }}|d t|� }}|d t|� }}i }g }�x�|jd��rx�|jd��rLtjd|�}ttt |j
� ��\} }
} g } xRt | �D ]F}
|d t|� }}tjd|�}ttt |j
� ��\}}| j ||f� q�W |d t|� }}| || |
f< |d t|� }}q�W tjd |�}ttt |j
� ��\}}g }x^t |�D ]R}
|d t|� }}tjd
|�}ttt |j
� ��\} } }
|| |
f } |j | � �q~W |j |� |d t|� }}|d t|� }}q�W || _
i }tjd |�}t |jd��}x�t |�D ]�}|d t|� }}tjd |�}|jd
�}ttt |jdddd���\}}}}|| }|d t|� }}tjd|�}i }t|jd��}xPt|�D ]D\}} t| �} x0t d�D ]$}| d|> @ �r�d||d | < �q�W �q�W ||f||< �q4W |d t|� }}|| _g } |d t|� }}tjd|�}t |jd��} xjt | �D ]^}|d t|� }}tjd|�}|j
� \}}t |�}|dk�r�d}nt|�}| j ||f� �qpW |d t|� }}| | _|d t|� }}|d t|� }}tjd|�}t |jd��}|d t|� }}|d t|� }}tjd|�}t |jd��} |d t|� }}tjd|�}t |jd��} | | _|d t|� }}y|d t|� }}W n tk
�r� Y nX dS )a� Parse the .c file written by pgen. (Internal)
The file looks as follows. The first two lines are always this:
#include "pgenheaders.h"
#include "grammar.h"
After that come four blocks:
1) one or more state definitions
2) a table defining dfas
3) a table defining labels
4) a struct defining the grammar
A state definition has the following form:
- one or more arc arrays, each of the form:
static arc arcs_<n>_<m>[<k>] = {
{<i>, <j>},
...
};
- followed by a state array, of the form:
static state states_<s>[<t>] = {
{<k>, arcs_<n>_<m>},
...
};
zCan't open %s: %sFNr r z static arc z)static arc arcs_(\d+)_(\d+)\[(\d+)\] = {$z\s+{(\d+), (\d+)},$z'static state states_(\d+)\[(\d+)\] = {$z \s+{(\d+), arcs_(\d+)_(\d+)},$z static dfa dfas\[(\d+)\] = {$z0\s+{(\d+), "(\w+)", (\d+), (\d+), states_(\d+),$� � � � z\s+("(?:\\\d\d\d)*")},$� z!static label labels\[(\d+)\] = {$z\s+{(\d+), (0|"\w+")},$�0z
\s+(\d+),$z\s+{(\d+), labels},$z \s+(\d+)$)r
r r �next�
startswithr r �list�mapr r �range�append�states�group�eval� enumerate�ord�dfas�labels�start�
StopIteration)!r r r r r r Zallarcsr) r �n�m�kZarcs�_�i�j�s�t�stater. Zndfasr r �x�y�z�firstZ rawbitset�cZbyter/ Znlabelsr0 r r r
r T s�
"
zConverter.parse_graminit_cc C s\ i | _ i | _xJt| j�D ]<\}\}}|tjkrB|dk rB|| j |< q|dkr|| j|< qW dS )z1Create additional useful structures. (Internal).N)�keywords�tokensr, r/ r �NAME)r Zilabel�type�valuer r r
r � s zConverter.finish_offN)�__name__�
__module__� __qualname__�__doc__r r r r r r r r
r $ s &r )rH r Zpgen2r r ZGrammarr r r r r
�<module> s __pycache__/literals.cpython-36.opt-2.pyc 0000644 00000002365 15046736237 0014204 0 ustar 00 3
\O � @ sL d dl Z dddddddd d
d d �
Zd
d� Zdd� Zdd� ZedkrHe� dS )� N��� �
�
� � �'�"�\)
�a�b�f�n�r�t�vr r
r c C s� | j dd�\}}tj|�}|d k r&|S |jd�r�|dd � }t|�dk rTtd| ��yt|d�}W q� tk
r� td| ��Y q�X n0yt|d�}W n tk
r� td| ��Y nX t|�S ) Nr � �x� z!invalid hex string escape ('\%s')� � z#invalid octal string escape ('\%s'))�group�simple_escapes�get�
startswith�len�
ValueError�int�chr)�m�all�tail�escZhexes�i� r% �./usr/lib64/python3.6/lib2to3/pgen2/literals.py�escape s"
r' c C sH | d }| d d� |d kr$|d }| t |�t |� � } tjdt| �S )Nr � z)\\(\'|\"|\\|[abfnrtv]|x.{0,2}|[0-7]{1,3}))r �re�subr' )�s�qr% r% r& �
evalString( s
r- c C sD x>t d�D ]2} t| �}t|�}t|�}||kr
t| |||� q
W d S )N� )�ranger �reprr- �print)r$ �cr+ �er% r% r& �test2 s r4 �__main__)r) r r' r- r4 �__name__r% r% r% r& �<module> s
tokenize.py 0000644 00000051077 15046736237 0006775 0 ustar 00 # Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006 Python Software Foundation.
# All rights reserved.
"""Tokenization help for Python programs.
generate_tokens(readline) is a generator that breaks a stream of
text into Python tokens. It accepts a readline-like method which is called
repeatedly to get the next line of input (or "" for EOF). It generates
5-tuples with these members:
the token type (see token.py)
the token (a string)
the starting (row, column) indices of the token (a 2-tuple of ints)
the ending (row, column) indices of the token (a 2-tuple of ints)
the original line (string)
It is designed to match the working of the Python tokenizer exactly, except
that it produces COMMENT tokens for comments and gives type OP for all
operators
Older entry points
tokenize_loop(readline, tokeneater)
tokenize(readline, tokeneater=printtoken)
are the same, except instead of generating tokens, tokeneater is a callback
function to which the 5 fields described above are passed as 5 arguments,
each time a new token is found."""
__author__ = 'Ka-Ping Yee <ping@lfw.org>'
__credits__ = \
'GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, Skip Montanaro'
import string, re
from codecs import BOM_UTF8, lookup
from lib2to3.pgen2.token import *
from . import token
__all__ = [x for x in dir(token) if x[0] != '_'] + ["tokenize",
"generate_tokens", "untokenize"]
del token
try:
bytes
except NameError:
# Support bytes type in Python <= 2.5, so 2to3 turns itself into
# valid Python 3 code.
bytes = str
def group(*choices): return '(' + '|'.join(choices) + ')'
def any(*choices): return group(*choices) + '*'
def maybe(*choices): return group(*choices) + '?'
def _combinations(*l):
return set(
x + y for x in l for y in l + ("",) if x.casefold() != y.casefold()
)
Whitespace = r'[ \f\t]*'
Comment = r'#[^\r\n]*'
Ignore = Whitespace + any(r'\\\r?\n' + Whitespace) + maybe(Comment)
Name = r'\w+'
Binnumber = r'0[bB]_?[01]+(?:_[01]+)*'
Hexnumber = r'0[xX]_?[\da-fA-F]+(?:_[\da-fA-F]+)*[lL]?'
Octnumber = r'0[oO]?_?[0-7]+(?:_[0-7]+)*[lL]?'
Decnumber = group(r'[1-9]\d*(?:_\d+)*[lL]?', '0[lL]?')
Intnumber = group(Binnumber, Hexnumber, Octnumber, Decnumber)
Exponent = r'[eE][-+]?\d+(?:_\d+)*'
Pointfloat = group(r'\d+(?:_\d+)*\.(?:\d+(?:_\d+)*)?', r'\.\d+(?:_\d+)*') + maybe(Exponent)
Expfloat = r'\d+(?:_\d+)*' + Exponent
Floatnumber = group(Pointfloat, Expfloat)
Imagnumber = group(r'\d+(?:_\d+)*[jJ]', Floatnumber + r'[jJ]')
Number = group(Imagnumber, Floatnumber, Intnumber)
# Tail end of ' string.
Single = r"[^'\\]*(?:\\.[^'\\]*)*'"
# Tail end of " string.
Double = r'[^"\\]*(?:\\.[^"\\]*)*"'
# Tail end of ''' string.
Single3 = r"[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''"
# Tail end of """ string.
Double3 = r'[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""'
_litprefix = r"(?:[uUrRbBfF]|[rR][fFbB]|[fFbBuU][rR])?"
Triple = group(_litprefix + "'''", _litprefix + '"""')
# Single-line ' or " string.
String = group(_litprefix + r"'[^\n'\\]*(?:\\.[^\n'\\]*)*'",
_litprefix + r'"[^\n"\\]*(?:\\.[^\n"\\]*)*"')
# Because of leftmost-then-longest match semantics, be sure to put the
# longest operators first (e.g., if = came before ==, == would get
# recognized as two instances of =).
Operator = group(r"\*\*=?", r">>=?", r"<<=?", r"<>", r"!=",
r"//=?", r"->",
r"[+\-*/%&@|^=<>]=?",
r"~")
Bracket = '[][(){}]'
Special = group(r'\r?\n', r':=', r'[:;.,`@]')
Funny = group(Operator, Bracket, Special)
PlainToken = group(Number, Funny, String, Name)
Token = Ignore + PlainToken
# First (or only) line of ' or " string.
ContStr = group(_litprefix + r"'[^\n'\\]*(?:\\.[^\n'\\]*)*" +
group("'", r'\\\r?\n'),
_litprefix + r'"[^\n"\\]*(?:\\.[^\n"\\]*)*' +
group('"', r'\\\r?\n'))
PseudoExtras = group(r'\\\r?\n', Comment, Triple)
PseudoToken = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name)
tokenprog, pseudoprog, single3prog, double3prog = map(
re.compile, (Token, PseudoToken, Single3, Double3))
_strprefixes = (
_combinations('r', 'R', 'f', 'F') |
_combinations('r', 'R', 'b', 'B') |
{'u', 'U', 'ur', 'uR', 'Ur', 'UR'}
)
endprogs = {"'": re.compile(Single), '"': re.compile(Double),
"'''": single3prog, '"""': double3prog,
**{f"{prefix}'''": single3prog for prefix in _strprefixes},
**{f'{prefix}"""': double3prog for prefix in _strprefixes},
**{prefix: None for prefix in _strprefixes}}
triple_quoted = (
{"'''", '"""'} |
{f"{prefix}'''" for prefix in _strprefixes} |
{f'{prefix}"""' for prefix in _strprefixes}
)
single_quoted = (
{"'", '"'} |
{f"{prefix}'" for prefix in _strprefixes} |
{f'{prefix}"' for prefix in _strprefixes}
)
tabsize = 8
class TokenError(Exception): pass
class StopTokenizing(Exception): pass
def printtoken(type, token, xxx_todo_changeme, xxx_todo_changeme1, line): # for testing
(srow, scol) = xxx_todo_changeme
(erow, ecol) = xxx_todo_changeme1
print("%d,%d-%d,%d:\t%s\t%s" % \
(srow, scol, erow, ecol, tok_name[type], repr(token)))
def tokenize(readline, tokeneater=printtoken):
"""
The tokenize() function accepts two parameters: one representing the
input stream, and one providing an output mechanism for tokenize().
The first parameter, readline, must be a callable object which provides
the same interface as the readline() method of built-in file objects.
Each call to the function should return one line of input as a string.
The second parameter, tokeneater, must also be a callable object. It is
called once for each token, with five arguments, corresponding to the
tuples generated by generate_tokens().
"""
try:
tokenize_loop(readline, tokeneater)
except StopTokenizing:
pass
# backwards compatible interface
def tokenize_loop(readline, tokeneater):
for token_info in generate_tokens(readline):
tokeneater(*token_info)
class Untokenizer:
def __init__(self):
self.tokens = []
self.prev_row = 1
self.prev_col = 0
def add_whitespace(self, start):
row, col = start
assert row <= self.prev_row
col_offset = col - self.prev_col
if col_offset:
self.tokens.append(" " * col_offset)
def untokenize(self, iterable):
for t in iterable:
if len(t) == 2:
self.compat(t, iterable)
break
tok_type, token, start, end, line = t
self.add_whitespace(start)
self.tokens.append(token)
self.prev_row, self.prev_col = end
if tok_type in (NEWLINE, NL):
self.prev_row += 1
self.prev_col = 0
return "".join(self.tokens)
def compat(self, token, iterable):
startline = False
indents = []
toks_append = self.tokens.append
toknum, tokval = token
if toknum in (NAME, NUMBER):
tokval += ' '
if toknum in (NEWLINE, NL):
startline = True
for tok in iterable:
toknum, tokval = tok[:2]
if toknum in (NAME, NUMBER, ASYNC, AWAIT):
tokval += ' '
if toknum == INDENT:
indents.append(tokval)
continue
elif toknum == DEDENT:
indents.pop()
continue
elif toknum in (NEWLINE, NL):
startline = True
elif startline and indents:
toks_append(indents[-1])
startline = False
toks_append(tokval)
cookie_re = re.compile(r'^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)', re.ASCII)
blank_re = re.compile(br'^[ \t\f]*(?:[#\r\n]|$)', re.ASCII)
def _get_normal_name(orig_enc):
"""Imitates get_normal_name in tokenizer.c."""
# Only care about the first 12 characters.
enc = orig_enc[:12].lower().replace("_", "-")
if enc == "utf-8" or enc.startswith("utf-8-"):
return "utf-8"
if enc in ("latin-1", "iso-8859-1", "iso-latin-1") or \
enc.startswith(("latin-1-", "iso-8859-1-", "iso-latin-1-")):
return "iso-8859-1"
return orig_enc
def detect_encoding(readline):
"""
The detect_encoding() function is used to detect the encoding that should
be used to decode a Python source file. It requires one argument, readline,
in the same way as the tokenize() generator.
It will call readline a maximum of twice, and return the encoding used
(as a string) and a list of any lines (left as bytes) it has read
in.
It detects the encoding from the presence of a utf-8 bom or an encoding
cookie as specified in pep-0263. If both a bom and a cookie are present, but
disagree, a SyntaxError will be raised. If the encoding cookie is an invalid
charset, raise a SyntaxError. Note that if a utf-8 bom is found,
'utf-8-sig' is returned.
If no encoding is specified, then the default of 'utf-8' will be returned.
"""
bom_found = False
encoding = None
default = 'utf-8'
def read_or_stop():
try:
return readline()
except StopIteration:
return bytes()
def find_cookie(line):
try:
line_string = line.decode('ascii')
except UnicodeDecodeError:
return None
match = cookie_re.match(line_string)
if not match:
return None
encoding = _get_normal_name(match.group(1))
try:
codec = lookup(encoding)
except LookupError:
# This behaviour mimics the Python interpreter
raise SyntaxError("unknown encoding: " + encoding)
if bom_found:
if codec.name != 'utf-8':
# This behaviour mimics the Python interpreter
raise SyntaxError('encoding problem: utf-8')
encoding += '-sig'
return encoding
first = read_or_stop()
if first.startswith(BOM_UTF8):
bom_found = True
first = first[3:]
default = 'utf-8-sig'
if not first:
return default, []
encoding = find_cookie(first)
if encoding:
return encoding, [first]
if not blank_re.match(first):
return default, [first]
second = read_or_stop()
if not second:
return default, [first]
encoding = find_cookie(second)
if encoding:
return encoding, [first, second]
return default, [first, second]
def untokenize(iterable):
"""Transform tokens back into Python source code.
Each element returned by the iterable must be a token sequence
with at least two elements, a token number and token value. If
only two tokens are passed, the resulting output is poor.
Round-trip invariant for full input:
Untokenized source will match input source exactly
Round-trip invariant for limited input:
# Output text will tokenize the back to the input
t1 = [tok[:2] for tok in generate_tokens(f.readline)]
newcode = untokenize(t1)
readline = iter(newcode.splitlines(1)).next
t2 = [tok[:2] for tokin generate_tokens(readline)]
assert t1 == t2
"""
ut = Untokenizer()
return ut.untokenize(iterable)
def generate_tokens(readline):
"""
The generate_tokens() generator requires one argument, readline, which
must be a callable object which provides the same interface as the
readline() method of built-in file objects. Each call to the function
should return one line of input as a string. Alternately, readline
can be a callable function terminating with StopIteration:
readline = open(myfile).next # Example of alternate readline
The generator produces 5-tuples with these members: the token type; the
token string; a 2-tuple (srow, scol) of ints specifying the row and
column where the token begins in the source; a 2-tuple (erow, ecol) of
ints specifying the row and column where the token ends in the source;
and the line on which the token was found. The line passed is the
physical line.
"""
lnum = parenlev = continued = 0
contstr, needcont = '', 0
contline = None
indents = [0]
# 'stashed' and 'async_*' are used for async/await parsing
stashed = None
async_def = False
async_def_indent = 0
async_def_nl = False
while 1: # loop over lines in stream
try:
line = readline()
except StopIteration:
line = ''
lnum = lnum + 1
pos, max = 0, len(line)
if contstr: # continued string
if not line:
raise TokenError("EOF in multi-line string", strstart)
endmatch = endprog.match(line)
if endmatch:
pos = end = endmatch.end(0)
yield (STRING, contstr + line[:end],
strstart, (lnum, end), contline + line)
contstr, needcont = '', 0
contline = None
elif needcont and line[-2:] != '\\\n' and line[-3:] != '\\\r\n':
yield (ERRORTOKEN, contstr + line,
strstart, (lnum, len(line)), contline)
contstr = ''
contline = None
continue
else:
contstr = contstr + line
contline = contline + line
continue
elif parenlev == 0 and not continued: # new statement
if not line: break
column = 0
while pos < max: # measure leading whitespace
if line[pos] == ' ': column = column + 1
elif line[pos] == '\t': column = (column//tabsize + 1)*tabsize
elif line[pos] == '\f': column = 0
else: break
pos = pos + 1
if pos == max: break
if stashed:
yield stashed
stashed = None
if line[pos] in '#\r\n': # skip comments or blank lines
if line[pos] == '#':
comment_token = line[pos:].rstrip('\r\n')
nl_pos = pos + len(comment_token)
yield (COMMENT, comment_token,
(lnum, pos), (lnum, pos + len(comment_token)), line)
yield (NL, line[nl_pos:],
(lnum, nl_pos), (lnum, len(line)), line)
else:
yield ((NL, COMMENT)[line[pos] == '#'], line[pos:],
(lnum, pos), (lnum, len(line)), line)
continue
if column > indents[-1]: # count indents or dedents
indents.append(column)
yield (INDENT, line[:pos], (lnum, 0), (lnum, pos), line)
while column < indents[-1]:
if column not in indents:
raise IndentationError(
"unindent does not match any outer indentation level",
("<tokenize>", lnum, pos, line))
indents = indents[:-1]
if async_def and async_def_indent >= indents[-1]:
async_def = False
async_def_nl = False
async_def_indent = 0
yield (DEDENT, '', (lnum, pos), (lnum, pos), line)
if async_def and async_def_nl and async_def_indent >= indents[-1]:
async_def = False
async_def_nl = False
async_def_indent = 0
else: # continued statement
if not line:
raise TokenError("EOF in multi-line statement", (lnum, 0))
continued = 0
while pos < max:
pseudomatch = pseudoprog.match(line, pos)
if pseudomatch: # scan for tokens
start, end = pseudomatch.span(1)
spos, epos, pos = (lnum, start), (lnum, end), end
token, initial = line[start:end], line[start]
if initial in string.digits or \
(initial == '.' and token != '.'): # ordinary number
yield (NUMBER, token, spos, epos, line)
elif initial in '\r\n':
newline = NEWLINE
if parenlev > 0:
newline = NL
elif async_def:
async_def_nl = True
if stashed:
yield stashed
stashed = None
yield (newline, token, spos, epos, line)
elif initial == '#':
assert not token.endswith("\n")
if stashed:
yield stashed
stashed = None
yield (COMMENT, token, spos, epos, line)
elif token in triple_quoted:
endprog = endprogs[token]
endmatch = endprog.match(line, pos)
if endmatch: # all on one line
pos = endmatch.end(0)
token = line[start:pos]
if stashed:
yield stashed
stashed = None
yield (STRING, token, spos, (lnum, pos), line)
else:
strstart = (lnum, start) # multiple lines
contstr = line[start:]
contline = line
break
elif initial in single_quoted or \
token[:2] in single_quoted or \
token[:3] in single_quoted:
if token[-1] == '\n': # continued string
strstart = (lnum, start)
endprog = (endprogs[initial] or endprogs[token[1]] or
endprogs[token[2]])
contstr, needcont = line[start:], 1
contline = line
break
else: # ordinary string
if stashed:
yield stashed
stashed = None
yield (STRING, token, spos, epos, line)
elif initial.isidentifier(): # ordinary name
if token in ('async', 'await'):
if async_def:
yield (ASYNC if token == 'async' else AWAIT,
token, spos, epos, line)
continue
tok = (NAME, token, spos, epos, line)
if token == 'async' and not stashed:
stashed = tok
continue
if token == 'def':
if (stashed
and stashed[0] == NAME
and stashed[1] == 'async'):
async_def = True
async_def_indent = indents[-1]
yield (ASYNC, stashed[1],
stashed[2], stashed[3],
stashed[4])
stashed = None
if stashed:
yield stashed
stashed = None
yield tok
elif initial == '\\': # continued stmt
# This yield is new; needed for better idempotency:
if stashed:
yield stashed
stashed = None
yield (NL, token, spos, (lnum, pos), line)
continued = 1
else:
if initial in '([{': parenlev = parenlev + 1
elif initial in ')]}': parenlev = parenlev - 1
if stashed:
yield stashed
stashed = None
yield (OP, token, spos, epos, line)
else:
yield (ERRORTOKEN, line[pos],
(lnum, pos), (lnum, pos+1), line)
pos = pos + 1
if stashed:
yield stashed
stashed = None
for indent in indents[1:]: # pop remaining indent levels
yield (DEDENT, '', (lnum, 0), (lnum, 0), '')
yield (ENDMARKER, '', (lnum, 0), (lnum, 0), '')
if __name__ == '__main__': # testing
import sys
if len(sys.argv) > 1: tokenize(open(sys.argv[1]).readline)
else: tokenize(sys.stdin.readline)
literals.py 0000644 00000003143 15046736237 0006753 0 ustar 00 # Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved.
# Licensed to PSF under a Contributor Agreement.
"""Safely evaluate Python string literals without using eval()."""
import re
simple_escapes = {"a": "\a",
"b": "\b",
"f": "\f",
"n": "\n",
"r": "\r",
"t": "\t",
"v": "\v",
"'": "'",
'"': '"',
"\\": "\\"}
def escape(m):
all, tail = m.group(0, 1)
assert all.startswith("\\")
esc = simple_escapes.get(tail)
if esc is not None:
return esc
if tail.startswith("x"):
hexes = tail[1:]
if len(hexes) < 2:
raise ValueError("invalid hex string escape ('\\%s')" % tail)
try:
i = int(hexes, 16)
except ValueError:
raise ValueError("invalid hex string escape ('\\%s')" % tail) from None
else:
try:
i = int(tail, 8)
except ValueError:
raise ValueError("invalid octal string escape ('\\%s')" % tail) from None
return chr(i)
def evalString(s):
assert s.startswith("'") or s.startswith('"'), repr(s[:1])
q = s[0]
if s[:3] == q*3:
q = q*3
assert s.endswith(q), repr(s[-len(q):])
assert len(s) >= 2*len(q)
s = s[len(q):-len(q)]
return re.sub(r"\\(\'|\"|\\|[abfnrtv]|x.{0,2}|[0-7]{1,3})", escape, s)
def test():
for i in range(256):
c = chr(i)
s = repr(c)
e = evalString(s)
if e != c:
print(i, c, s, e)
if __name__ == "__main__":
test()
pgen.py 0000644 00000032764 15046736237 0006100 0 ustar 00 # Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved.
# Licensed to PSF under a Contributor Agreement.
# Pgen imports
from . import grammar, token, tokenize
class PgenGrammar(grammar.Grammar):
pass
class ParserGenerator(object):
def __init__(self, filename, stream=None):
close_stream = None
if stream is None:
stream = open(filename)
close_stream = stream.close
self.filename = filename
self.stream = stream
self.generator = tokenize.generate_tokens(stream.readline)
self.gettoken() # Initialize lookahead
self.dfas, self.startsymbol = self.parse()
if close_stream is not None:
close_stream()
self.first = {} # map from symbol name to set of tokens
self.addfirstsets()
def make_grammar(self):
c = PgenGrammar()
names = list(self.dfas.keys())
names.sort()
names.remove(self.startsymbol)
names.insert(0, self.startsymbol)
for name in names:
i = 256 + len(c.symbol2number)
c.symbol2number[name] = i
c.number2symbol[i] = name
for name in names:
dfa = self.dfas[name]
states = []
for state in dfa:
arcs = []
for label, next in sorted(state.arcs.items()):
arcs.append((self.make_label(c, label), dfa.index(next)))
if state.isfinal:
arcs.append((0, dfa.index(state)))
states.append(arcs)
c.states.append(states)
c.dfas[c.symbol2number[name]] = (states, self.make_first(c, name))
c.start = c.symbol2number[self.startsymbol]
return c
def make_first(self, c, name):
rawfirst = self.first[name]
first = {}
for label in sorted(rawfirst):
ilabel = self.make_label(c, label)
##assert ilabel not in first # XXX failed on <> ... !=
first[ilabel] = 1
return first
def make_label(self, c, label):
# XXX Maybe this should be a method on a subclass of converter?
ilabel = len(c.labels)
if label[0].isalpha():
# Either a symbol name or a named token
if label in c.symbol2number:
# A symbol name (a non-terminal)
if label in c.symbol2label:
return c.symbol2label[label]
else:
c.labels.append((c.symbol2number[label], None))
c.symbol2label[label] = ilabel
return ilabel
else:
# A named token (NAME, NUMBER, STRING)
itoken = getattr(token, label, None)
assert isinstance(itoken, int), label
assert itoken in token.tok_name, label
if itoken in c.tokens:
return c.tokens[itoken]
else:
c.labels.append((itoken, None))
c.tokens[itoken] = ilabel
return ilabel
else:
# Either a keyword or an operator
assert label[0] in ('"', "'"), label
value = eval(label)
if value[0].isalpha():
# A keyword
if value in c.keywords:
return c.keywords[value]
else:
c.labels.append((token.NAME, value))
c.keywords[value] = ilabel
return ilabel
else:
# An operator (any non-numeric token)
itoken = grammar.opmap[value] # Fails if unknown token
if itoken in c.tokens:
return c.tokens[itoken]
else:
c.labels.append((itoken, None))
c.tokens[itoken] = ilabel
return ilabel
def addfirstsets(self):
names = list(self.dfas.keys())
names.sort()
for name in names:
if name not in self.first:
self.calcfirst(name)
#print name, self.first[name].keys()
def calcfirst(self, name):
dfa = self.dfas[name]
self.first[name] = None # dummy to detect left recursion
state = dfa[0]
totalset = {}
overlapcheck = {}
for label, next in state.arcs.items():
if label in self.dfas:
if label in self.first:
fset = self.first[label]
if fset is None:
raise ValueError("recursion for rule %r" % name)
else:
self.calcfirst(label)
fset = self.first[label]
totalset.update(fset)
overlapcheck[label] = fset
else:
totalset[label] = 1
overlapcheck[label] = {label: 1}
inverse = {}
for label, itsfirst in overlapcheck.items():
for symbol in itsfirst:
if symbol in inverse:
raise ValueError("rule %s is ambiguous; %s is in the"
" first sets of %s as well as %s" %
(name, symbol, label, inverse[symbol]))
inverse[symbol] = label
self.first[name] = totalset
def parse(self):
dfas = {}
startsymbol = None
# MSTART: (NEWLINE | RULE)* ENDMARKER
while self.type != token.ENDMARKER:
while self.type == token.NEWLINE:
self.gettoken()
# RULE: NAME ':' RHS NEWLINE
name = self.expect(token.NAME)
self.expect(token.OP, ":")
a, z = self.parse_rhs()
self.expect(token.NEWLINE)
#self.dump_nfa(name, a, z)
dfa = self.make_dfa(a, z)
#self.dump_dfa(name, dfa)
oldlen = len(dfa)
self.simplify_dfa(dfa)
newlen = len(dfa)
dfas[name] = dfa
#print name, oldlen, newlen
if startsymbol is None:
startsymbol = name
return dfas, startsymbol
def make_dfa(self, start, finish):
# To turn an NFA into a DFA, we define the states of the DFA
# to correspond to *sets* of states of the NFA. Then do some
# state reduction. Let's represent sets as dicts with 1 for
# values.
assert isinstance(start, NFAState)
assert isinstance(finish, NFAState)
def closure(state):
base = {}
addclosure(state, base)
return base
def addclosure(state, base):
assert isinstance(state, NFAState)
if state in base:
return
base[state] = 1
for label, next in state.arcs:
if label is None:
addclosure(next, base)
states = [DFAState(closure(start), finish)]
for state in states: # NB states grows while we're iterating
arcs = {}
for nfastate in state.nfaset:
for label, next in nfastate.arcs:
if label is not None:
addclosure(next, arcs.setdefault(label, {}))
for label, nfaset in sorted(arcs.items()):
for st in states:
if st.nfaset == nfaset:
break
else:
st = DFAState(nfaset, finish)
states.append(st)
state.addarc(st, label)
return states # List of DFAState instances; first one is start
def dump_nfa(self, name, start, finish):
print("Dump of NFA for", name)
todo = [start]
for i, state in enumerate(todo):
print(" State", i, state is finish and "(final)" or "")
for label, next in state.arcs:
if next in todo:
j = todo.index(next)
else:
j = len(todo)
todo.append(next)
if label is None:
print(" -> %d" % j)
else:
print(" %s -> %d" % (label, j))
def dump_dfa(self, name, dfa):
print("Dump of DFA for", name)
for i, state in enumerate(dfa):
print(" State", i, state.isfinal and "(final)" or "")
for label, next in sorted(state.arcs.items()):
print(" %s -> %d" % (label, dfa.index(next)))
def simplify_dfa(self, dfa):
# This is not theoretically optimal, but works well enough.
# Algorithm: repeatedly look for two states that have the same
# set of arcs (same labels pointing to the same nodes) and
# unify them, until things stop changing.
# dfa is a list of DFAState instances
changes = True
while changes:
changes = False
for i, state_i in enumerate(dfa):
for j in range(i+1, len(dfa)):
state_j = dfa[j]
if state_i == state_j:
#print " unify", i, j
del dfa[j]
for state in dfa:
state.unifystate(state_j, state_i)
changes = True
break
def parse_rhs(self):
# RHS: ALT ('|' ALT)*
a, z = self.parse_alt()
if self.value != "|":
return a, z
else:
aa = NFAState()
zz = NFAState()
aa.addarc(a)
z.addarc(zz)
while self.value == "|":
self.gettoken()
a, z = self.parse_alt()
aa.addarc(a)
z.addarc(zz)
return aa, zz
def parse_alt(self):
# ALT: ITEM+
a, b = self.parse_item()
while (self.value in ("(", "[") or
self.type in (token.NAME, token.STRING)):
c, d = self.parse_item()
b.addarc(c)
b = d
return a, b
def parse_item(self):
# ITEM: '[' RHS ']' | ATOM ['+' | '*']
if self.value == "[":
self.gettoken()
a, z = self.parse_rhs()
self.expect(token.OP, "]")
a.addarc(z)
return a, z
else:
a, z = self.parse_atom()
value = self.value
if value not in ("+", "*"):
return a, z
self.gettoken()
z.addarc(a)
if value == "+":
return a, z
else:
return a, a
def parse_atom(self):
# ATOM: '(' RHS ')' | NAME | STRING
if self.value == "(":
self.gettoken()
a, z = self.parse_rhs()
self.expect(token.OP, ")")
return a, z
elif self.type in (token.NAME, token.STRING):
a = NFAState()
z = NFAState()
a.addarc(z, self.value)
self.gettoken()
return a, z
else:
self.raise_error("expected (...) or NAME or STRING, got %s/%s",
self.type, self.value)
def expect(self, type, value=None):
if self.type != type or (value is not None and self.value != value):
self.raise_error("expected %s/%s, got %s/%s",
type, value, self.type, self.value)
value = self.value
self.gettoken()
return value
def gettoken(self):
tup = next(self.generator)
while tup[0] in (tokenize.COMMENT, tokenize.NL):
tup = next(self.generator)
self.type, self.value, self.begin, self.end, self.line = tup
#print token.tok_name[self.type], repr(self.value)
def raise_error(self, msg, *args):
if args:
try:
msg = msg % args
except:
msg = " ".join([msg] + list(map(str, args)))
raise SyntaxError(msg, (self.filename, self.end[0],
self.end[1], self.line))
class NFAState(object):
def __init__(self):
self.arcs = [] # list of (label, NFAState) pairs
def addarc(self, next, label=None):
assert label is None or isinstance(label, str)
assert isinstance(next, NFAState)
self.arcs.append((label, next))
class DFAState(object):
def __init__(self, nfaset, final):
assert isinstance(nfaset, dict)
assert isinstance(next(iter(nfaset)), NFAState)
assert isinstance(final, NFAState)
self.nfaset = nfaset
self.isfinal = final in nfaset
self.arcs = {} # map from label to DFAState
def addarc(self, next, label):
assert isinstance(label, str)
assert label not in self.arcs
assert isinstance(next, DFAState)
self.arcs[label] = next
def unifystate(self, old, new):
for label, next in self.arcs.items():
if next is old:
self.arcs[label] = new
def __eq__(self, other):
# Equality test -- ignore the nfaset instance variable
assert isinstance(other, DFAState)
if self.isfinal != other.isfinal:
return False
# Can't just return self.arcs == other.arcs, because that
# would invoke this method recursively, with cycles...
if len(self.arcs) != len(other.arcs):
return False
for label, next in self.arcs.items():
if next is not other.arcs.get(label):
return False
return True
__hash__ = None # For Py3 compatibility.
def generate_grammar(filename="Grammar.txt"):
p = ParserGenerator(filename)
return p.make_grammar()
driver.py 0000644 00000013521 15046736237 0006430 0 ustar 00 # Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved.
# Licensed to PSF under a Contributor Agreement.
# Modifications:
# Copyright 2006 Google, Inc. All Rights Reserved.
# Licensed to PSF under a Contributor Agreement.
"""Parser driver.
This provides a high-level interface to parse a file into a syntax tree.
"""
__author__ = "Guido van Rossum <guido@python.org>"
__all__ = ["Driver", "load_grammar"]
# Python imports
import io
import os
import logging
import pkgutil
import sys
# Pgen imports
from . import grammar, parse, token, tokenize, pgen
class Driver(object):
def __init__(self, grammar, convert=None, logger=None):
self.grammar = grammar
if logger is None:
logger = logging.getLogger()
self.logger = logger
self.convert = convert
def parse_tokens(self, tokens, debug=False):
"""Parse a series of tokens and return the syntax tree."""
# XXX Move the prefix computation into a wrapper around tokenize.
p = parse.Parser(self.grammar, self.convert)
p.setup()
lineno = 1
column = 0
type = value = start = end = line_text = None
prefix = ""
for quintuple in tokens:
type, value, start, end, line_text = quintuple
if start != (lineno, column):
assert (lineno, column) <= start, ((lineno, column), start)
s_lineno, s_column = start
if lineno < s_lineno:
prefix += "\n" * (s_lineno - lineno)
lineno = s_lineno
column = 0
if column < s_column:
prefix += line_text[column:s_column]
column = s_column
if type in (tokenize.COMMENT, tokenize.NL):
prefix += value
lineno, column = end
if value.endswith("\n"):
lineno += 1
column = 0
continue
if type == token.OP:
type = grammar.opmap[value]
if debug:
self.logger.debug("%s %r (prefix=%r)",
token.tok_name[type], value, prefix)
if p.addtoken(type, value, (prefix, start)):
if debug:
self.logger.debug("Stop.")
break
prefix = ""
lineno, column = end
if value.endswith("\n"):
lineno += 1
column = 0
else:
# We never broke out -- EOF is too soon (how can this happen???)
raise parse.ParseError("incomplete input",
type, value, (prefix, start))
return p.rootnode
def parse_stream_raw(self, stream, debug=False):
"""Parse a stream and return the syntax tree."""
tokens = tokenize.generate_tokens(stream.readline)
return self.parse_tokens(tokens, debug)
def parse_stream(self, stream, debug=False):
"""Parse a stream and return the syntax tree."""
return self.parse_stream_raw(stream, debug)
def parse_file(self, filename, encoding=None, debug=False):
"""Parse a file and return the syntax tree."""
with io.open(filename, "r", encoding=encoding) as stream:
return self.parse_stream(stream, debug)
def parse_string(self, text, debug=False):
"""Parse a string and return the syntax tree."""
tokens = tokenize.generate_tokens(io.StringIO(text).readline)
return self.parse_tokens(tokens, debug)
def _generate_pickle_name(gt):
head, tail = os.path.splitext(gt)
if tail == ".txt":
tail = ""
return head + tail + ".".join(map(str, sys.version_info)) + ".pickle"
def load_grammar(gt="Grammar.txt", gp=None,
save=True, force=False, logger=None):
"""Load the grammar (maybe from a pickle)."""
if logger is None:
logger = logging.getLogger()
gp = _generate_pickle_name(gt) if gp is None else gp
if force or not _newer(gp, gt):
logger.info("Generating grammar tables from %s", gt)
g = pgen.generate_grammar(gt)
if save:
logger.info("Writing grammar tables to %s", gp)
try:
g.dump(gp)
except OSError as e:
logger.info("Writing failed: %s", e)
else:
g = grammar.Grammar()
g.load(gp)
return g
def _newer(a, b):
"""Inquire whether file a was written since file b."""
if not os.path.exists(a):
return False
if not os.path.exists(b):
return True
return os.path.getmtime(a) >= os.path.getmtime(b)
def load_packaged_grammar(package, grammar_source):
"""Normally, loads a pickled grammar by doing
pkgutil.get_data(package, pickled_grammar)
where *pickled_grammar* is computed from *grammar_source* by adding the
Python version and using a ``.pickle`` extension.
However, if *grammar_source* is an extant file, load_grammar(grammar_source)
is called instead. This facilitates using a packaged grammar file when needed
but preserves load_grammar's automatic regeneration behavior when possible.
"""
if os.path.isfile(grammar_source):
return load_grammar(grammar_source)
pickled_name = _generate_pickle_name(os.path.basename(grammar_source))
data = pkgutil.get_data(package, pickled_name)
g = grammar.Grammar()
g.loads(data)
return g
def main(*args):
"""Main program, when run as a script: produce grammar pickle files.
Calls load_grammar for each argument, a path to a grammar text file.
"""
if not args:
args = sys.argv[1:]
logging.basicConfig(level=logging.INFO, stream=sys.stdout,
format='%(message)s')
for gt in args:
load_grammar(gt, save=True, force=True)
return True
if __name__ == "__main__":
sys.exit(int(not main()))
grammar.py 0000644 00000012635 15046736237 0006570 0 ustar 00 # Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved.
# Licensed to PSF under a Contributor Agreement.
"""This module defines the data structures used to represent a grammar.
These are a bit arcane because they are derived from the data
structures used by Python's 'pgen' parser generator.
There's also a table here mapping operators to their names in the
token module; the Python tokenize module reports all operators as the
fallback token code OP, but the parser needs the actual token code.
"""
# Python imports
import pickle
# Local imports
from . import token
class Grammar(object):
"""Pgen parsing tables conversion class.
Once initialized, this class supplies the grammar tables for the
parsing engine implemented by parse.py. The parsing engine
accesses the instance variables directly. The class here does not
provide initialization of the tables; several subclasses exist to
do this (see the conv and pgen modules).
The load() method reads the tables from a pickle file, which is
much faster than the other ways offered by subclasses. The pickle
file is written by calling dump() (after loading the grammar
tables using a subclass). The report() method prints a readable
representation of the tables to stdout, for debugging.
The instance variables are as follows:
symbol2number -- a dict mapping symbol names to numbers. Symbol
numbers are always 256 or higher, to distinguish
them from token numbers, which are between 0 and
255 (inclusive).
number2symbol -- a dict mapping numbers to symbol names;
these two are each other's inverse.
states -- a list of DFAs, where each DFA is a list of
states, each state is a list of arcs, and each
arc is a (i, j) pair where i is a label and j is
a state number. The DFA number is the index into
this list. (This name is slightly confusing.)
Final states are represented by a special arc of
the form (0, j) where j is its own state number.
dfas -- a dict mapping symbol numbers to (DFA, first)
pairs, where DFA is an item from the states list
above, and first is a set of tokens that can
begin this grammar rule (represented by a dict
whose values are always 1).
labels -- a list of (x, y) pairs where x is either a token
number or a symbol number, and y is either None
or a string; the strings are keywords. The label
number is the index in this list; label numbers
are used to mark state transitions (arcs) in the
DFAs.
start -- the number of the grammar's start symbol.
keywords -- a dict mapping keyword strings to arc labels.
tokens -- a dict mapping token numbers to arc labels.
"""
def __init__(self):
self.symbol2number = {}
self.number2symbol = {}
self.states = []
self.dfas = {}
self.labels = [(0, "EMPTY")]
self.keywords = {}
self.tokens = {}
self.symbol2label = {}
self.start = 256
def dump(self, filename):
"""Dump the grammar tables to a pickle file."""
with open(filename, "wb") as f:
pickle.dump(self.__dict__, f, pickle.HIGHEST_PROTOCOL)
def load(self, filename):
"""Load the grammar tables from a pickle file."""
with open(filename, "rb") as f:
d = pickle.load(f)
self.__dict__.update(d)
def loads(self, pkl):
"""Load the grammar tables from a pickle bytes object."""
self.__dict__.update(pickle.loads(pkl))
def copy(self):
"""
Copy the grammar.
"""
new = self.__class__()
for dict_attr in ("symbol2number", "number2symbol", "dfas", "keywords",
"tokens", "symbol2label"):
setattr(new, dict_attr, getattr(self, dict_attr).copy())
new.labels = self.labels[:]
new.states = self.states[:]
new.start = self.start
return new
def report(self):
"""Dump the grammar tables to standard output, for debugging."""
from pprint import pprint
print("s2n")
pprint(self.symbol2number)
print("n2s")
pprint(self.number2symbol)
print("states")
pprint(self.states)
print("dfas")
pprint(self.dfas)
print("labels")
pprint(self.labels)
print("start", self.start)
# Map from operator to number (since tokenize doesn't do this)
opmap_raw = """
( LPAR
) RPAR
[ LSQB
] RSQB
: COLON
, COMMA
; SEMI
+ PLUS
- MINUS
* STAR
/ SLASH
| VBAR
& AMPER
< LESS
> GREATER
= EQUAL
. DOT
% PERCENT
` BACKQUOTE
{ LBRACE
} RBRACE
@ AT
@= ATEQUAL
== EQEQUAL
!= NOTEQUAL
<> NOTEQUAL
<= LESSEQUAL
>= GREATEREQUAL
~ TILDE
^ CIRCUMFLEX
<< LEFTSHIFT
>> RIGHTSHIFT
** DOUBLESTAR
+= PLUSEQUAL
-= MINEQUAL
*= STAREQUAL
/= SLASHEQUAL
%= PERCENTEQUAL
&= AMPEREQUAL
|= VBAREQUAL
^= CIRCUMFLEXEQUAL
<<= LEFTSHIFTEQUAL
>>= RIGHTSHIFTEQUAL
**= DOUBLESTAREQUAL
// DOUBLESLASH
//= DOUBLESLASHEQUAL
-> RARROW
:= COLONEQUAL
"""
opmap = {}
for line in opmap_raw.splitlines():
if line:
op, name = line.split()
opmap[op] = getattr(token, name)
token.py 0000755 00000002424 15046736237 0006260 0 ustar 00 #! /usr/bin/python3.8
"""Token constants (from "token.h")."""
# Taken from Python (r53757) and modified to include some tokens
# originally monkeypatched in by pgen2.tokenize
#--start constants--
ENDMARKER = 0
NAME = 1
NUMBER = 2
STRING = 3
NEWLINE = 4
INDENT = 5
DEDENT = 6
LPAR = 7
RPAR = 8
LSQB = 9
RSQB = 10
COLON = 11
COMMA = 12
SEMI = 13
PLUS = 14
MINUS = 15
STAR = 16
SLASH = 17
VBAR = 18
AMPER = 19
LESS = 20
GREATER = 21
EQUAL = 22
DOT = 23
PERCENT = 24
BACKQUOTE = 25
LBRACE = 26
RBRACE = 27
EQEQUAL = 28
NOTEQUAL = 29
LESSEQUAL = 30
GREATEREQUAL = 31
TILDE = 32
CIRCUMFLEX = 33
LEFTSHIFT = 34
RIGHTSHIFT = 35
DOUBLESTAR = 36
PLUSEQUAL = 37
MINEQUAL = 38
STAREQUAL = 39
SLASHEQUAL = 40
PERCENTEQUAL = 41
AMPEREQUAL = 42
VBAREQUAL = 43
CIRCUMFLEXEQUAL = 44
LEFTSHIFTEQUAL = 45
RIGHTSHIFTEQUAL = 46
DOUBLESTAREQUAL = 47
DOUBLESLASH = 48
DOUBLESLASHEQUAL = 49
AT = 50
ATEQUAL = 51
OP = 52
COMMENT = 53
NL = 54
RARROW = 55
AWAIT = 56
ASYNC = 57
ERRORTOKEN = 58
COLONEQUAL = 59
N_TOKENS = 60
NT_OFFSET = 256
#--end constants--
tok_name = {}
for _name, _value in list(globals().items()):
if type(_value) is type(0):
tok_name[_value] = _name
def ISTERMINAL(x):
return x < NT_OFFSET
def ISNONTERMINAL(x):
return x >= NT_OFFSET
def ISEOF(x):
return x == ENDMARKER
pgen.pyc 0000644 00000027716 15047135540 0006233 0 ustar 00 �
{ fc @ s� d d l m Z m Z m Z d e j f d � � YZ d e f d � � YZ d e f d � � YZ d e f d � � YZ d
d � Z
d S(
i ( t grammart tokent tokenizet PgenGrammarc B s e Z RS( ( t __name__t
__module__( ( ( s* /usr/lib64/python2.7/lib2to3/pgen2/pgen.pyR s t ParserGeneratorc B s� e Z d d � Z d � Z d � Z d � Z d � Z d � Z d � Z d � Z
d � Z d � Z d
� Z
d � Z d � Z d
� Z d � Z d d � Z d � Z d � Z RS( c C s� d } | d k r* t | � } | j } n | | _ | | _ t j | j � | _ | j � | j
� \ | _ | _ | d k r� | � n i | _
| j � d S( N( t Nonet opent closet filenamet streamR t generate_tokenst readlinet generatort gettokent parset dfast startsymbolt firstt addfirstsets( t selfR
R t close_stream( ( s* /usr/lib64/python2.7/lib2to3/pgen2/pgen.pyt __init__ s
c C s� t � } | j j � } | j � | j | j � | j d | j � x; | D]3 } d t | j � } | | j | <| | j | <qL Wx� | D]� } | j | } g } x� | D]� } g } xK t
| j j � � D]4 \ } }
| j
| j | | � | j |
� f � q� W| j r,| j
d | j | � f � n | j
| � q� W| j j
| � | | j | | � f | j | j | <q� W| j | j | _ | S( Ni i ( R R t keyst sortt removeR t insertt lent
symbol2numbert
number2symbolt sortedt arcst iteritemst appendt
make_labelt indext isfinalt statest
make_firstt start( R t ct namest namet it dfaR& t stateR t labelt next( ( s* /usr/lib64/python2.7/lib2to3/pgen2/pgen.pyt make_grammar s.
", *c C sJ | j | } i } x0 t | � D]" } | j | | � } d | | <q W| S( Ni ( R R R# ( R R) R+ t rawfirstR R/ t ilabel( ( s* /usr/lib64/python2.7/lib2to3/pgen2/pgen.pyR' 4 s
c C s� t | j � } | d j � r | | j k ry | | j k rH | j | S| j j | j | d f � | | j | <| Sq�t t | d � } t | t
t f � s� t | � � | t j
k s� t | � � | | j k r� | j | S| j j | d f � | | j | <| Sn� | d d k s't | � � t | � } | d j � r�| | j k r]| j | S| j j t j | f � | | j | <| SnN t j | } | | j k r�| j | S| j j | d f � | | j | <| Sd S( Ni t "t '( R4 R5 ( R t labelst isalphaR t symbol2labelR" R t getattrR t
isinstancet intt longt AssertionErrort tok_namet tokenst evalt keywordst NAMER t opmap( R R) R/ R3 t itokent value( ( s* /usr/lib64/python2.7/lib2to3/pgen2/pgen.pyR# = s<
!
c C sM | j j � } | j � x- | D]% } | | j k r | j | � q q Wd S( N( R R R R t calcfirst( R R* R+ ( ( s* /usr/lib64/python2.7/lib2to3/pgen2/pgen.pyR k s
c C sj | j | } d | j | <| d } i } i } x� | j j � D]� \ } } | | j k r� | | j k r� | j | } | d k r� t d | � � q� n | j | � | j | } | j | � | | | <q@ d | | <i d | 6| | <q@ Wi } xd | j � D]V \ } }
xG |
D]? } | | k rGt d | | | | | f � � n | | | <qWq� W| | j | <d S( Ni s recursion for rule %ri sA rule %s is ambiguous; %s is in the first sets of %s as well as %s( R R R R R! t
ValueErrorRF t update( R R+ R- R. t totalsett overlapcheckR/ R0 t fsett inverset itsfirstt symbol( ( s* /usr/lib64/python2.7/lib2to3/pgen2/pgen.pyRF s s2
c C s� i } d } x� | j t j k r� x | j t j k rC | j � q$ W| j t j � } | j t j d � | j � \ } } | j t j � | j
| | � } t | � } | j | � t | � } | | | <| d k r | } q q W| | f S( Nt :(
R t typeR t ENDMARKERt NEWLINER t expectRB t OPt parse_rhst make_dfaR t simplify_dfa( R R R R+ t at zR- t oldlent newlen( ( s* /usr/lib64/python2.7/lib2to3/pgen2/pgen.pyR � s"
c sD t | t � s t � t | t � s* t � � f d � } � f d � � t | | � | � g } x� | D]� } i } xS | j D]H } x? | j D]4 \ } } | d k r� � | | j | i � � q� q� Wq} Wxp t | j � � D]\ \ } }
x= | D] } | j |
k r� Pq� q� Wt |
| � } | j
| � | j | | � q� Wqg W| S( Nc s i } � | | � | S( N( ( R. t base( t
addclosure( s* /usr/lib64/python2.7/lib2to3/pgen2/pgen.pyt closure� s
c si t | t � s t � | | k r% d Sd | | <x3 | j D]( \ } } | d k r9 � | | � q9 q9 Wd S( Ni ( R: t NFAStateR= R R ( R. R\ R/ R0 ( R] ( s* /usr/lib64/python2.7/lib2to3/pgen2/pgen.pyR] � s
( R: R_ R= t DFAStatet nfasetR R t
setdefaultR R! R" t addarc( R R( t finishR^ R& R. R t nfastateR/ R0 Ra t st( ( R] s* /usr/lib64/python2.7/lib2to3/pgen2/pgen.pyRV � s&
$
c
C s� d G| GH| g } x� t | � D]� \ } } d G| G| | k rE d pH d GHxu | j D]j \ } } | | k r~ | j | � } n t | � } | j | � | d k r� d | GHqT d | | f GHqT Wq Wd S( Ns Dump of NFA fors States (final)t s -> %ds %s -> %d( t enumerateR R$ R R" R (
R R+ R( Rd t todoR, R. R/ R0 t j( ( s* /usr/lib64/python2.7/lib2to3/pgen2/pgen.pyt dump_nfa� s
c C s� d G| GHxt t | � D]f \ } } d G| G| j r9 d p< d GHx; t | j j � � D]$ \ } } d | | j | � f GHqT Wq Wd S( Ns Dump of DFA fors States (final)Rg s %s -> %d( Rh R% R R R! R$ ( R R+ R- R, R. R/ R0 ( ( s* /usr/lib64/python2.7/lib2to3/pgen2/pgen.pyt dump_dfa� s
"c C s� t } x� | r� t } x� t | � D]x \ } } xi t | d t | � � D]N } | | } | | k rH | | =x | D] } | j | | � qr Wt } PqH qH Wq" Wq Wd S( Ni ( t Truet FalseRh t rangeR t
unifystate( R R- t changesR, t state_iRj t state_jR. ( ( s* /usr/lib64/python2.7/lib2to3/pgen2/pgen.pyRW � s
c C s� | j � \ } } | j d k r+ | | f St � } t � } | j | � | j | � xI | j d k r� | j � | j � \ } } | j | � | j | � qZ W| | f Sd S( Nt |( t parse_altRE R_ Rc R ( R RX RY t aat zz( ( s* /usr/lib64/python2.7/lib2to3/pgen2/pgen.pyRU � s
c C sr | j � \ } } xS | j d k s? | j t j t j f k rg | j � \ } } | j | � | } q W| | f S( Nt (t [( Rx Ry ( t
parse_itemRE RP R RB t STRINGRc ( R RX t bR) t d( ( s* /usr/lib64/python2.7/lib2to3/pgen2/pgen.pyRu
s
c C s� | j d k rU | j � | j � \ } } | j t j d � | j | � | | f S| j � \ } } | j } | d k r� | | f S| j � | j | � | d k r� | | f S| | f Sd S( NRy t ]t +t *( R R� ( RE R RU RS R RT Rc t
parse_atom( R RX RY RE ( ( s* /usr/lib64/python2.7/lib2to3/pgen2/pgen.pyRz s
c C s� | j d k rH | j � | j � \ } } | j t j d � | | f S| j t j t j f k r� t � } t � } | j
| | j � | j � | | f S| j d | j | j � d S( NRx t )s+ expected (...) or NAME or STRING, got %s/%s( RE R RU RS R RT RP RB R{ R_ Rc t raise_error( R RX RY ( ( s* /usr/lib64/python2.7/lib2to3/pgen2/pgen.pyR� ( s
c C sc | j | k s* | d k rL | j | k rL | j d | | | j | j � n | j } | j � | S( Ns expected %s/%s, got %s/%s( RP R RE R� R ( R RP RE ( ( s* /usr/lib64/python2.7/lib2to3/pgen2/pgen.pyRS 9 s *
c C si | j j � } x/ | d t j t j f k r@ | j j � } q W| \ | _ | _ | _ | _ | _ d S( Ni (
R R0 R t COMMENTt NLRP RE t begint endt line( R t tup( ( s* /usr/lib64/python2.7/lib2to3/pgen2/pgen.pyR A s c G ss | r@ y | | } Wq@ d j | g t t | � � } q@ Xn t | | j | j d | j d | j f � � d S( Nt i i ( t joint mapt strt SyntaxErrorR
R� R� ( R t msgt args( ( s* /usr/lib64/python2.7/lib2to3/pgen2/pgen.pyR� H s &N( R R R R R1 R' R# R RF R RV Rk Rl RW RU Ru Rz R� RS R R� ( ( ( s* /usr/lib64/python2.7/lib2to3/pgen2/pgen.pyR
s$ . $
R_ c B s e Z d � Z d d � Z RS( c C s
g | _ d S( N( R ( R ( ( s* /usr/lib64/python2.7/lib2to3/pgen2/pgen.pyR S s c C sP | d k s! t | t � s! t � t | t � s6 t � | j j | | f � d S( N( R R: R� R= R_ R R" ( R R0 R/ ( ( s* /usr/lib64/python2.7/lib2to3/pgen2/pgen.pyRc V s !N( R R R R Rc ( ( ( s* /usr/lib64/python2.7/lib2to3/pgen2/pgen.pyR_ Q s R` c B s2 e Z d � Z d � Z d � Z d � Z d Z RS( c C sp t | t � s t � t t | � j � t � s6 t � t | t � sK t � | | _ | | k | _ i | _ d S( N( R: t dictR= t iterR0 R_ Ra R% R ( R Ra t final( ( s* /usr/lib64/python2.7/lib2to3/pgen2/pgen.pyR ] s ! c C sP t | t � s t � | | j k s* t � t | t � s? t � | | j | <d S( N( R: R� R= R R` ( R R0 R/ ( ( s* /usr/lib64/python2.7/lib2to3/pgen2/pgen.pyRc e s c C s@ x9 | j j � D]( \ } } | | k r | | j | <q q Wd S( N( R R! ( R t oldt newR/ R0 ( ( s* /usr/lib64/python2.7/lib2to3/pgen2/pgen.pyRp k s c C s� t | t � s t � | j | j k r+ t St | j � t | j � k rM t Sx9 | j j � D]( \ } } | | j j | � k r] t Sq] Wt S( N(
R: R` R= R% Rn R R R! t getRm ( R t otherR/ R0 ( ( s* /usr/lib64/python2.7/lib2to3/pgen2/pgen.pyt __eq__p s N( R R R Rc Rp R� R t __hash__( ( ( s* /usr/lib64/python2.7/lib2to3/pgen2/pgen.pyR` [ s
s Grammar.txtc C s t | � } | j � S( N( R R1 ( R
t p( ( s* /usr/lib64/python2.7/lib2to3/pgen2/pgen.pyt generate_grammar� s N( Rg R R R t GrammarR t objectR R_ R` R� ( ( ( s* /usr/lib64/python2.7/lib2to3/pgen2/pgen.pyt <module> s � H
% pgen.pyo 0000644 00000026551 15047135540 0006243 0 ustar 00 �
{ fc @ s� d d l m Z m Z m Z d e j f d � � YZ d e f d � � YZ d e f d � � YZ d e f d � � YZ d
d � Z
d S(
i ( t grammart tokent tokenizet PgenGrammarc B s e Z RS( ( t __name__t
__module__( ( ( s* /usr/lib64/python2.7/lib2to3/pgen2/pgen.pyR s t ParserGeneratorc B s� e Z d d � Z d � Z d � Z d � Z d � Z d � Z d � Z d � Z
d � Z d � Z d
� Z
d � Z d � Z d
� Z d � Z d d � Z d � Z d � Z RS( c C s� d } | d k r* t | � } | j } n | | _ | | _ t j | j � | _ | j � | j
� \ | _ | _ | d k r� | � n i | _
| j � d S( N( t Nonet opent closet filenamet streamR t generate_tokenst readlinet generatort gettokent parset dfast startsymbolt firstt addfirstsets( t selfR
R t close_stream( ( s* /usr/lib64/python2.7/lib2to3/pgen2/pgen.pyt __init__ s
c C s� t � } | j j � } | j � | j | j � | j d | j � x; | D]3 } d t | j � } | | j | <| | j | <qL Wx� | D]� } | j | } g } x� | D]� } g } xK t
| j j � � D]4 \ } }
| j
| j | | � | j |
� f � q� W| j r,| j
d | j | � f � n | j
| � q� W| j j
| � | | j | | � f | j | j | <q� W| j | j | _ | S( Ni i ( R R t keyst sortt removeR t insertt lent
symbol2numbert
number2symbolt sortedt arcst iteritemst appendt
make_labelt indext isfinalt statest
make_firstt start( R t ct namest namet it dfaR& t stateR t labelt next( ( s* /usr/lib64/python2.7/lib2to3/pgen2/pgen.pyt make_grammar s.
", *c C sJ | j | } i } x0 t | � D]" } | j | | � } d | | <q W| S( Ni ( R R R# ( R R) R+ t rawfirstR R/ t ilabel( ( s* /usr/lib64/python2.7/lib2to3/pgen2/pgen.pyR' 4 s
c C s� t | j � } | d j � r� | | j k ry | | j k rH | j | S| j j | j | d f � | | j | <| Sq�t t | d � } | | j k r� | j | S| j j | d f � | | j | <| Sn� t
| � } | d j � r2| | j k r| j | S| j j t j | f � | | j | <| SnN t
j | } | | j k rY| j | S| j j | d f � | | j | <| Sd S( Ni ( R t labelst isalphaR t symbol2labelR" R t getattrR t tokenst evalt keywordst NAMER t opmap( R R) R/ R3 t itokent value( ( s* /usr/lib64/python2.7/lib2to3/pgen2/pgen.pyR# = s6
c C sM | j j � } | j � x- | D]% } | | j k r | j | � q q Wd S( N( R R R R t calcfirst( R R* R+ ( ( s* /usr/lib64/python2.7/lib2to3/pgen2/pgen.pyR k s
c C sj | j | } d | j | <| d } i } i } x� | j j � D]� \ } } | | j k r� | | j k r� | j | } | d k r� t d | � � q� n | j | � | j | } | j | � | | | <q@ d | | <i d | 6| | <q@ Wi } xd | j � D]V \ } }
xG |
D]? } | | k rGt d | | | | | f � � n | | | <qWq� W| | j | <d S( Ni s recursion for rule %ri sA rule %s is ambiguous; %s is in the first sets of %s as well as %s( R R R R R! t
ValueErrorR? t update( R R+ R- R. t totalsett overlapcheckR/ R0 t fsett inverset itsfirstt symbol( ( s* /usr/lib64/python2.7/lib2to3/pgen2/pgen.pyR? s s2
c C s� i } d } x� | j t j k r� x | j t j k rC | j � q$ W| j t j � } | j t j d � | j � \ } } | j t j � | j
| | � } t | � } | j | � t | � } | | | <| d k r | } q q W| | f S( Nt :(
R t typeR t ENDMARKERt NEWLINER t expectR; t OPt parse_rhst make_dfaR t simplify_dfa( R R R R+ t at zR- t oldlent newlen( ( s* /usr/lib64/python2.7/lib2to3/pgen2/pgen.pyR � s"
c s � f d � } � f d � � t | | � | � g } x� | D]� } i } xS | j D]H } x? | j D]4 \ } } | d k rc � | | j | i � � qc qc WqS Wxp t | j � � D]\ \ } }
x= | D] } | j |
k r� Pq� q� Wt |
| � } | j | � | j | | � q� Wq= W| S( Nc s i } � | | � | S( N( ( R. t base( t
addclosure( s* /usr/lib64/python2.7/lib2to3/pgen2/pgen.pyt closure� s
c sT | | k r d Sd | | <x3 | j D]( \ } } | d k r$ � | | � q$ q$ Wd S( Ni ( R R ( R. RU R/ R0 ( RV ( s* /usr/lib64/python2.7/lib2to3/pgen2/pgen.pyRV � s
( t DFAStatet nfasetR R t
setdefaultR R! R" t addarc( R R( t finishRW R& R. R t nfastateR/ R0 RY t st( ( RV s* /usr/lib64/python2.7/lib2to3/pgen2/pgen.pyRO � s"
$
c
C s� d G| GH| g } x� t | � D]� \ } } d G| G| | k rE d pH d GHxu | j D]j \ } } | | k r~ | j | � } n t | � } | j | � | d k r� d | GHqT d | | f GHqT Wq Wd S( Ns Dump of NFA fors States (final)t s -> %ds %s -> %d( t enumerateR R$ R R" R (
R R+ R( R\ t todoR, R. R/ R0 t j( ( s* /usr/lib64/python2.7/lib2to3/pgen2/pgen.pyt dump_nfa� s
c C s� d G| GHxt t | � D]f \ } } d G| G| j r9 d p< d GHx; t | j j � � D]$ \ } } d | | j | � f GHqT Wq Wd S( Ns Dump of DFA fors States (final)R_ s %s -> %d( R` R% R R R! R$ ( R R+ R- R, R. R/ R0 ( ( s* /usr/lib64/python2.7/lib2to3/pgen2/pgen.pyt dump_dfa� s
"c C s� t } x� | r� t } x� t | � D]x \ } } xi t | d t | � � D]N } | | } | | k rH | | =x | D] } | j | | � qr Wt } PqH qH Wq" Wq Wd S( Ni ( t Truet FalseR` t rangeR t
unifystate( R R- t changesR, t state_iRb t state_jR. ( ( s* /usr/lib64/python2.7/lib2to3/pgen2/pgen.pyRP � s
c C s� | j � \ } } | j d k r+ | | f St � } t � } | j | � | j | � xI | j d k r� | j � | j � \ } } | j | � | j | � qZ W| | f Sd S( Nt |( t parse_altR> t NFAStateR[ R ( R RQ RR t aat zz( ( s* /usr/lib64/python2.7/lib2to3/pgen2/pgen.pyRN � s
c C sr | j � \ } } xS | j d k s? | j t j t j f k rg | j � \ } } | j | � | } q W| | f S( Nt (t [( Rq Rr ( t
parse_itemR> RI R R; t STRINGR[ ( R RQ t bR) t d( ( s* /usr/lib64/python2.7/lib2to3/pgen2/pgen.pyRm
s
c C s� | j d k rU | j � | j � \ } } | j t j d � | j | � | | f S| j � \ } } | j } | d k r� | | f S| j � | j | � | d k r� | | f S| | f Sd S( NRr t ]t +t *( Rx Ry ( R> R RN RL R RM R[ t
parse_atom( R RQ RR R> ( ( s* /usr/lib64/python2.7/lib2to3/pgen2/pgen.pyRs s
c C s� | j d k rH | j � | j � \ } } | j t j d � | | f S| j t j t j f k r� t � } t � } | j
| | j � | j � | | f S| j d | j | j � d S( NRq t )s+ expected (...) or NAME or STRING, got %s/%s( R> R RN RL R RM RI R; Rt Rn R[ t raise_error( R RQ RR ( ( s* /usr/lib64/python2.7/lib2to3/pgen2/pgen.pyRz ( s
c C sc | j | k s* | d k rL | j | k rL | j d | | | j | j � n | j } | j � | S( Ns expected %s/%s, got %s/%s( RI R R> R| R ( R RI R> ( ( s* /usr/lib64/python2.7/lib2to3/pgen2/pgen.pyRL 9 s *
c C si | j j � } x/ | d t j t j f k r@ | j j � } q W| \ | _ | _ | _ | _ | _ d S( Ni (
R R0 R t COMMENTt NLRI R> t begint endt line( R t tup( ( s* /usr/lib64/python2.7/lib2to3/pgen2/pgen.pyR A s c G ss | r@ y | | } Wq@ d j | g t t | � � } q@ Xn t | | j | j d | j d | j f � � d S( Nt i i ( t joint mapt strt SyntaxErrorR
R� R� ( R t msgt args( ( s* /usr/lib64/python2.7/lib2to3/pgen2/pgen.pyR| H s &N( R R R R R1 R' R# R R? R RO Rc Rd RP RN Rm Rs Rz RL R R| ( ( ( s* /usr/lib64/python2.7/lib2to3/pgen2/pgen.pyR
s$ . $
Rn c B s e Z d � Z d d � Z RS( c C s
g | _ d S( N( R ( R ( ( s* /usr/lib64/python2.7/lib2to3/pgen2/pgen.pyR S s c C s | j j | | f � d S( N( R R" ( R R0 R/ ( ( s* /usr/lib64/python2.7/lib2to3/pgen2/pgen.pyR[ V s N( R R R R R[ ( ( ( s* /usr/lib64/python2.7/lib2to3/pgen2/pgen.pyRn Q s RX c B s2 e Z d � Z d � Z d � Z d � Z d Z RS( c C s% | | _ | | k | _ i | _ d S( N( RY R% R ( R RY t final( ( s* /usr/lib64/python2.7/lib2to3/pgen2/pgen.pyR ] s c C s | | j | <d S( N( R ( R R0 R/ ( ( s* /usr/lib64/python2.7/lib2to3/pgen2/pgen.pyR[ e s c C s@ x9 | j j � D]( \ } } | | k r | | j | <q q Wd S( N( R R! ( R t oldt newR/ R0 ( ( s* /usr/lib64/python2.7/lib2to3/pgen2/pgen.pyRh k s c C sx | j | j k r t St | j � t | j � k r8 t Sx9 | j j � D]( \ } } | | j j | � k rH t SqH Wt S( N( R% Rf R R R! t getRe ( R t otherR/ R0 ( ( s* /usr/lib64/python2.7/lib2to3/pgen2/pgen.pyt __eq__p s N( R R R R[ Rh R� R t __hash__( ( ( s* /usr/lib64/python2.7/lib2to3/pgen2/pgen.pyRX [ s
s Grammar.txtc C s t | � } | j � S( N( R R1 ( R
t p( ( s* /usr/lib64/python2.7/lib2to3/pgen2/pgen.pyt generate_grammar� s N( R_ R R R t GrammarR t objectR Rn RX R� ( ( ( s* /usr/lib64/python2.7/lib2to3/pgen2/pgen.pyt <module> s � H
% conv.pyo 0000644 00000015575 15047135540 0006263 0 ustar 00 �
{ fc @ sE d Z d d l Z d d l m Z m Z d e j f d � � YZ d S( s� Convert graminit.[ch] spit out by pgen to Python code.
Pgen is the Python parser generator. It is useful to quickly create a
parser from a grammar file in Python's grammar notation. But I don't
want my parsers to be written in C (yet), so I'm translating the
parsing tables to Python data structures and writing a Python parse
engine.
Note that the token numbers are constants determined by the standard
Python tokenizer. The standard token module defines these numbers and
their names (the names are not used much). The token numbers are
hardcoded into the Python tokenizer and into pgen. A Python
implementation of the Python tokenizer is also available, in the
standard tokenize module.
On the other hand, symbol numbers (representing the grammar's
non-terminals) are assigned by pgen based on the actual grammar
input.
Note: this module is pretty much obsolete; the pgen module generates
equivalent grammar tables directly from the Grammar.txt input file
without having to invoke the Python pgen C program.
i����N( t grammart tokent Converterc B s2 e Z d Z d � Z d � Z d � Z d � Z RS( s2 Grammar subclass that reads classic pgen output files.
The run() method reads the tables as produced by the pgen parser
generator, typically contained in two C files, graminit.h and
graminit.c. The other methods are for internal use only.
See the base class for more documentation.
c C s( | j | � | j | � | j � d S( s< Load the grammar tables from the text files written by pgen.N( t parse_graminit_ht parse_graminit_ct
finish_off( t selft
graminit_ht
graminit_c( ( s* /usr/lib64/python2.7/lib2to3/pgen2/conv.pyt run/ s
c C s� y t | � } Wn# t k
r5 } d | | f GHt SXi | _ i | _ d } x� | D]� } | d 7} t j d | � } | r� | j � r� d | | | j � f GHqU | j � \ } } t | � } | | j | <| | j | <qU Wt
S( s� Parse the .h file written by pgen. (Internal)
This file is a sequence of #define statements defining the
nonterminals of the grammar as numbers. We build two tables
mapping the numbers to names and back.
s Can't open %s: %si i s ^#define\s+(\w+)\s+(\d+)$s %s(%s): can't parse %s( t opent IOErrort Falset
symbol2numbert
number2symbolt ret matcht stript groupst intt True( R t filenamet ft errt linenot linet mot symbolt number( ( s* /usr/lib64/python2.7/lib2to3/pgen2/conv.pyR 5 s&
c ! C sH y t | � } Wn# t k
r5 } d | | f GHt SXd } | d | j � } } | d | j � } } | d | j � } } i } g } x�| j d � r�x� | j d � r�t j d | � } t t | j � � \ } }
} g } xk t
| � D]] }
| d | j � } } t j d | � } t t | j � � \ } } | j | | f � q� W| d | j � } } | | | |
f <| d | j � } } q� Wt j d | � } t t | j � � \ } } g } xx t
| � D]j }
| d | j � } } t j d | � } t t | j � � \ } } }
| | |
f } | j | � q�W| j | � | d | j � } } | d | j � } } q� W| | _ i } t j d | � } t | j
d � � } x:t
| � D],} | d | j � } } t j d
| � } | j
d � } t t | j
d d d
d � � \ } } } } | | } | d | j � } } t j d | � } i } t | j
d � � } x` t | � D]R \ } } t | � } x7 t
d � D]) } | d | >@r�d | | d | <q�q�Wq�W| | f | | <q�W| d | j � } } | | _ g } | d | j � } } t j d | � } t | j
d � � } x� t
| � D]� } | d | j � } } t j d | � } | j � \ } } t | � } | d k r�d } n t | � } | j | | f � qhW| d | j � } } | | _ | d | j � } } | d | j � } } t j d | � } t | j
d � � } | d | j � } } | d | j � } } t j d | � } t | j
d � � } | d | j � } } t j d | � } t | j
d � � } | | _ | d | j � } } y | d | j � } } Wn t k
rCn Xd S( s� Parse the .c file written by pgen. (Internal)
The file looks as follows. The first two lines are always this:
#include "pgenheaders.h"
#include "grammar.h"
After that come four blocks:
1) one or more state definitions
2) a table defining dfas
3) a table defining labels
4) a struct defining the grammar
A state definition has the following form:
- one or more arc arrays, each of the form:
static arc arcs_<n>_<m>[<k>] = {
{<i>, <j>},
...
};
- followed by a state array, of the form:
static state states_<s>[<t>] = {
{<k>, arcs_<n>_<m>},
...
};
s Can't open %s: %si i s static arc s) static arc arcs_(\d+)_(\d+)\[(\d+)\] = {$s \s+{(\d+), (\d+)},$s' static state states_(\d+)\[(\d+)\] = {$s \s+{(\d+), arcs_(\d+)_(\d+)},$s static dfa dfas\[(\d+)\] = {$s0 \s+{(\d+), "(\w+)", (\d+), (\d+), states_(\d+),$i i i i s \s+("(?:\\\d\d\d)*")},$i s! static label labels\[(\d+)\] = {$s \s+{(\d+), (0|"\w+")},$t 0s
\s+(\d+),$s \s+{(\d+), labels},$s \s+(\d+)$N( R
R R t nextt
startswithR R t mapR R t ranget appendt statest groupt evalt enumeratet ordt dfast Nonet labelst startt
StopIteration(! R R R R R R t allarcsR# R t nt mt kt arcst _t it jt st tt stateR( t ndfasR R t xt yt zt firstt rawbitsett ct byteR* t nlabelsR+ ( ( s* /usr/lib64/python2.7/lib2to3/pgen2/conv.pyR T s�
-
c C s� i | _ i | _ xj t | j � D]Y \ } \ } } | t j k r_ | d k r_ | | j | <q" | d k r" | | j | <q" q" Wd S( s1 Create additional useful structures. (Internal).N( t keywordst tokensR& R* R t NAMER) ( R t ilabelt typet value( ( s* /usr/lib64/python2.7/lib2to3/pgen2/conv.pyR � s " ( t __name__t
__module__t __doc__R R R R ( ( ( s* /usr/lib64/python2.7/lib2to3/pgen2/conv.pyR $ s
�( RI R t pgen2R R t GrammarR ( ( ( s* /usr/lib64/python2.7/lib2to3/pgen2/conv.pyt <module> s parse.pyc 0000644 00000016104 15047135540 0006401 0 ustar 00 �
{ fc @ sF d Z d d l m Z d e f d � � YZ d e f d � � YZ d S( s� Parser engine for the grammar tables generated by pgen.
The grammar table must be loaded first.
See Parser/parser.c in the Python distribution for additional info on
how this parsing engine works.
i ( t tokent
ParseErrorc B s e Z d Z d � Z RS( s( Exception to signal the parser is stuck.c C sH t j | d | | | | f � | | _ | | _ | | _ | | _ d S( Ns! %s: type=%r, value=%r, context=%r( t Exceptiont __init__t msgt typet valuet context( t selfR R R R ( ( s+ /usr/lib64/python2.7/lib2to3/pgen2/parse.pyR s ( t __name__t
__module__t __doc__R ( ( ( s+ /usr/lib64/python2.7/lib2to3/pgen2/parse.pyR s t Parserc B sS e Z d Z d d � Z d d � Z d � Z d � Z d � Z d � Z d � Z
RS( s5 Parser engine.
The proper usage sequence is:
p = Parser(grammar, [converter]) # create instance
p.setup([start]) # prepare for parsing
<for each input token>:
if p.addtoken(...): # parse a token; may raise ParseError
break
root = p.rootnode # root of abstract syntax tree
A Parser instance may be reused by calling setup() repeatedly.
A Parser instance contains state pertaining to the current token
sequence, and should not be used concurrently by different threads
to parse separate token sequences.
See driver.py for how to get input tokens by tokenizing a file or
string.
Parsing is complete when addtoken() returns True; the root of the
abstract syntax tree can then be retrieved from the rootnode
instance variable. When a syntax error occurs, addtoken() raises
the ParseError exception. There is no error recovery; the parser
cannot be used after a syntax error was reported (but it can be
reinitialized by calling setup()).
c C s | | _ | p d � | _ d S( s� Constructor.
The grammar argument is a grammar.Grammar instance; see the
grammar module for more information.
The parser is not ready yet for parsing; you must call the
setup() method to get it started.
The optional convert argument is a function mapping concrete
syntax tree nodes to abstract syntax tree nodes. If not
given, no conversion is done and the syntax tree produced is
the concrete syntax tree. If given, it must be a function of
two arguments, the first being the grammar (a grammar.Grammar
instance), and the second being the concrete syntax tree node
to be converted. The syntax tree is converted from the bottom
up.
A concrete syntax tree node is a (type, value, context, nodes)
tuple, where type is the node type (a token or symbol number),
value is None for symbols and a string for tokens, context is
None or an opaque value used for error reporting (typically a
(lineno, offset) pair), and nodes is a list of children for
symbols, and None for tokens.
An abstract syntax tree node may be anything; this is entirely
up to the converter function.
c S s | S( N( ( t grammart node( ( s+ /usr/lib64/python2.7/lib2to3/pgen2/parse.pyt <lambda>W t N( R
t convert( R R
R ( ( s+ /usr/lib64/python2.7/lib2to3/pgen2/parse.pyR 9 s c C sk | d k r | j j } n | d d g f } | j j | d | f } | g | _ d | _ t � | _ d S( s� Prepare for parsing.
This *must* be called before starting to parse.
The optional argument is an alternative start symbol; it
defaults to the grammar's start symbol.
You can use a Parser instance to parse any number of programs;
each time you call setup() the parser is reset to an initial
state determined by the (implicit or explicit) start symbol.
i N( t NoneR
t startt dfast stackt rootnodet sett
used_names( R R t newnodet
stackentry( ( s+ /usr/lib64/python2.7/lib2to3/pgen2/parse.pyt setupY s
c C s� | j | | | � } x�t r�| j d \ } } } | \ } } | | }
xq|
D]\ } } | j j | \ }
} | | k r
|
d k s� t � | j | | | | � | } xV | | d | f g k r| j � | j s� t S| j d \ } } } | \ } } q� Wt S|
d k rQ | j j |
} | \ } } | | k re| j
|
| j j |
| | � PqeqQ qQ Wd | f |
k r�| j � | j s�t d | | | � � q�q t d | | | � � q Wd S( s< Add a token; return True iff this is the end of the program.i����i i s too much inputs bad inputN( t classifyt TrueR R
t labelst AssertionErrort shiftt popt FalseR t pushR ( R R R R t ilabelt dfat stateR t statest firstt arcst it newstatet tt vt itsdfat itsstatest itsfirst( ( s+ /usr/lib64/python2.7/lib2to3/pgen2/parse.pyt addtokenq s<
c C s� | t j k rG | j j | � | j j j | � } | d k rG | Sn | j j j | � } | d k r� t d | | | � � n | S( s& Turn a token into a label. (Internal)s bad tokenN(
R t NAMER t addR
t keywordst getR t tokensR ( R R R R R$ ( ( s+ /usr/lib64/python2.7/lib2to3/pgen2/parse.pyR � s c C sw | j d \ } } } | | | d f } | j | j | � } | d k r] | d j | � n | | | f | j d <d S( s Shift a token. (Internal)i����N( R R R R
t append( R R R R+ R R% R& R R ( ( s+ /usr/lib64/python2.7/lib2to3/pgen2/parse.pyR � s c C s[ | j d \ } } } | d | g f } | | | f | j d <| j j | d | f � d S( s Push a nonterminal. (Internal)i����i N( R R R7 ( R R t newdfaR+ R R% R& R R ( ( s+ /usr/lib64/python2.7/lib2to3/pgen2/parse.pyR# � s c C s� | j j � \ } } } | j | j | � } | d k r� | j rl | j d \ } } } | d j | � q� | | _ | j | j _ n d S( s Pop a nonterminal. (Internal)i����N( R R! R R
R R7 R R ( R t popdfat popstatet popnodeR R% R& R ( ( s+ /usr/lib64/python2.7/lib2to3/pgen2/parse.pyR! � s N( R R
R R R R R1 R R R# R! ( ( ( s+ /usr/lib64/python2.7/lib2to3/pgen2/parse.pyR s 0 N( R R R R R t objectR ( ( ( s+ /usr/lib64/python2.7/lib2to3/pgen2/parse.pyt <module> s __init__.pyo 0000644 00000000256 15047135540 0007043 0 ustar 00 �
{ fc @ s
d Z d S( s The pgen2 package.N( t __doc__( ( ( s. /usr/lib64/python2.7/lib2to3/pgen2/__init__.pyt <module> t tokenize.pyc 0000644 00000041113 15047135540 0007115 0 ustar 00 �
{ fc @ s d Z d Z d Z d d l Z d d l Z d d l m Z m Z d d l Td d l m
Z
g e e
� D] Z e d d
k rg e ^ qg d d d
g Z
[
y e Wn e k
r� e Z n Xd � Z d � Z d � Z d Z d Z e e d e � e e � Z d Z d Z d Z d Z d Z e e e e e � Z d Z e d d � e e � Z d e Z e e e � Z e d e d � Z! e e! e e � Z" d Z# d Z$ d! Z% d" Z& e d# d$ � Z' e d% d& � Z( e d' d( d) d* d+ d, d- d. d/ � Z) d0 Z* e d1 d2 � Z+ e e) e* e+ � Z, e e" e, e( e � Z- e e- Z. e d3 e d4 d � d5 e d6 d � � Z/ e d e e' � Z0 e e e0 e" e, e/ e � Z1 e2 e j3 e. e1 e% e& f � \ Z4 Z5 Z6 Z7 i& e j3 e# � d4 6e j3 e$ � d6 6e6 d7 6e7 d8 6e6 d9 6e7 d: 6e6 d; 6e7 d< 6e6 d= 6e7 d> 6e6 d? 6e7 d@ 6e6 dA 6e7 dB 6e6 dC 6e7 dD 6e6 dE 6e7 dF 6e6 dG 6e7 dH 6e6 dI 6e7 dJ 6e6 dK 6e7 dL 6e6 dM 6e7 dN 6e6 dO 6e7 dP 6e6 dQ 6e7 dR 6e6 dS 6e7 dT 6d dU 6d dV 6d dW 6d dX 6d dY 6d dZ 6Z9 i Z: x d� D] Z; e; e: e; <q�Wi Z< x d� D] Z; e; e< e; <qWdw Z= dx e> f dy � � YZ? dz e> f d{ � � YZ@ d| � ZA eA d} � ZB d~ � ZC d d� d� � � YZD e j3 d� � ZE e j3 d� � ZF d� � ZG d� � ZH d� � ZI d� � ZJ eK d� k rd d lL ZL eM eL jN � d k reB eO eL jN d � jP � qeB eL jQ jP � n d S(� s� Tokenization help for Python programs.
generate_tokens(readline) is a generator that breaks a stream of
text into Python tokens. It accepts a readline-like method which is called
repeatedly to get the next line of input (or "" for EOF). It generates
5-tuples with these members:
the token type (see token.py)
the token (a string)
the starting (row, column) indices of the token (a 2-tuple of ints)
the ending (row, column) indices of the token (a 2-tuple of ints)
the original line (string)
It is designed to match the working of the Python tokenizer exactly, except
that it produces COMMENT tokens for comments and gives type OP for all
operators
Older entry points
tokenize_loop(readline, tokeneater)
tokenize(readline, tokeneater=printtoken)
are the same, except instead of generating tokens, tokeneater is a callback
function to which the 5 fields described above are passed as 5 arguments,
each time a new token is found.s Ka-Ping Yee <ping@lfw.org>s@ GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, Skip Montanaroi����N( t BOM_UTF8t lookup( t *i ( t tokeni t _t tokenizet generate_tokenst
untokenizec G s d d j | � d S( Nt (t |t )( t join( t choices( ( s. /usr/lib64/python2.7/lib2to3/pgen2/tokenize.pyt group0 t c G s t | � d S( NR ( R
( R ( ( s. /usr/lib64/python2.7/lib2to3/pgen2/tokenize.pyt any1 R c G s t | � d S( Nt ?( R
( R ( ( s. /usr/lib64/python2.7/lib2to3/pgen2/tokenize.pyt maybe2 R s [ \f\t]*s #[^\r\n]*s \\\r?\ns [a-zA-Z_]\w*s
0[bB][01]*s 0[xX][\da-fA-F]*[lL]?s 0[oO]?[0-7]*[lL]?s
[1-9]\d*[lL]?s [eE][-+]?\d+s \d+\.\d*s \.\d+s \d+s \d+[jJ]s [jJ]s [^'\\]*(?:\\.[^'\\]*)*'s [^"\\]*(?:\\.[^"\\]*)*"s% [^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''s% [^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""s [ubUB]?[rR]?'''s [ubUB]?[rR]?"""s& [uU]?[rR]?'[^\n'\\]*(?:\\.[^\n'\\]*)*'s& [uU]?[rR]?"[^\n"\\]*(?:\\.[^\n"\\]*)*"s \*\*=?s >>=?s <<=?s <>s !=s //=?s ->s [+\-*/%&@|^=<>]=?t ~s [][(){}]s \r?\ns [:;.,`@]s' [uUbB]?[rR]?'[^\n'\\]*(?:\\.[^\n'\\]*)*t 's' [uUbB]?[rR]?"[^\n"\\]*(?:\\.[^\n"\\]*)*t "s '''s """s r'''s r"""s u'''s u"""s b'''s b"""s ur'''s ur"""s br'''s br"""s R'''s R"""s U'''s U"""s B'''s B"""s uR'''s uR"""s Ur'''s Ur"""s UR'''s UR"""s bR'''s bR"""s Br'''s Br"""s BR'''s BR"""t rt Rt ut Ut bt Bs r's r"s R's R"s u's u"s U's U"s b's b"s B's B"s ur's ur"s Ur's Ur"s uR's uR"s UR's UR"s br's br"s Br's Br"s bR's bR"s BR's BR"i t
TokenErrorc B s e Z RS( ( t __name__t
__module__( ( ( s. /usr/lib64/python2.7/lib2to3/pgen2/tokenize.pyR � s t StopTokenizingc B s e Z RS( ( R R ( ( ( s. /usr/lib64/python2.7/lib2to3/pgen2/tokenize.pyR � s c C sA | \ } } | \ } } d | | | | t | t | � f GHd S( Ns %d,%d-%d,%d: %s %s( t tok_namet repr( t typeR t startt endt linet srowt scolt erowt ecol( ( s. /usr/lib64/python2.7/lib2to3/pgen2/tokenize.pyt
printtoken� s c C s) y t | | � Wn t k
r$ n Xd S( s:
The tokenize() function accepts two parameters: one representing the
input stream, and one providing an output mechanism for tokenize().
The first parameter, readline, must be a callable object which provides
the same interface as the readline() method of built-in file objects.
Each call to the function should return one line of input as a string.
The second parameter, tokeneater, must also be a callable object. It is
called once for each token, with five arguments, corresponding to the
tuples generated by generate_tokens().
N( t
tokenize_loopR ( t readlinet
tokeneater( ( s. /usr/lib64/python2.7/lib2to3/pgen2/tokenize.pyR � s
c C s% x t | � D] } | | � q
Wd S( N( R ( R+ R, t
token_info( ( s. /usr/lib64/python2.7/lib2to3/pgen2/tokenize.pyR* � s t Untokenizerc B s, e Z d � Z d � Z d � Z d � Z RS( c C s g | _ d | _ d | _ d S( Ni i ( t tokenst prev_rowt prev_col( t self( ( s. /usr/lib64/python2.7/lib2to3/pgen2/tokenize.pyt __init__� s c C sO | \ } } | | j k s! t � | | j } | rK | j j d | � n d S( Nt ( R0 t AssertionErrorR1 R/ t append( R2 R" t rowt colt
col_offset( ( s. /usr/lib64/python2.7/lib2to3/pgen2/tokenize.pyt add_whitespace� s
c C s� x� | D]� } t | � d k r3 | j | | � Pn | \ } } } } } | j | � | j j | � | \ | _ | _ | t t f k r | j d 7_ d | _ q q Wd j | j � S( Ni i i R (
t lent compatR: R/ R6 R0 R1 t NEWLINEt NLR ( R2 t iterablet tt tok_typeR R" R# R$ ( ( s. /usr/lib64/python2.7/lib2to3/pgen2/tokenize.pyR � s
c C s% t } g } | j j } | \ } } | t t f k rC | d 7} n | t t f k r^ t } n x� | D]� } | d \ } } | t t f k r� | d 7} n | t k r� | j | � qe nZ | t k r� | j
� qe n> | t t f k r� t } n# | r| r| | d � t } n | | � qe Wd S( NR4 i i����( t FalseR/ R6 t NAMEt NUMBERR= R> t Truet INDENTt DEDENTt pop( R2 R R? t startlinet indentst toks_appendt toknumt tokvalt tok( ( s. /usr/lib64/python2.7/lib2to3/pgen2/tokenize.pyR< � s0
( R R R3 R: R R< ( ( ( s. /usr/lib64/python2.7/lib2to3/pgen2/tokenize.pyR. � s s& ^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)s ^[ \t\f]*(?:[#\r\n]|$)c C s^ | d j � j d d � } | d k s7 | j d � r; d S| d k sV | j d
� rZ d S| S( s( Imitates get_normal_name in tokenizer.c.i R t -s utf-8s utf-8-s latin-1s
iso-8859-1s iso-latin-1s latin-1-s iso-8859-1-s iso-latin-1-( s latin-1s
iso-8859-1s iso-latin-1( s latin-1-s iso-8859-1-s iso-latin-1-( t lowert replacet
startswith( t orig_enct enc( ( s. /usr/lib64/python2.7/lib2to3/pgen2/tokenize.pyt _get_normal_name� s c s� t � d } d } � f d � } � f d � } | � } | j t � ra t � | d } d } n | sq | g f S| | � } | r� | | g f St j | � s� | | g f S| � } | s� | | g f S| | � } | r� | | | g f S| | | g f S( s
The detect_encoding() function is used to detect the encoding that should
be used to decode a Python source file. It requires one argument, readline,
in the same way as the tokenize() generator.
It will call readline a maximum of twice, and return the encoding used
(as a string) and a list of any lines (left as bytes) it has read
in.
It detects the encoding from the presence of a utf-8 bom or an encoding
cookie as specified in pep-0263. If both a bom and a cookie are present, but
disagree, a SyntaxError will be raised. If the encoding cookie is an invalid
charset, raise a SyntaxError. Note that if a utf-8 bom is found,
'utf-8-sig' is returned.
If no encoding is specified, then the default of 'utf-8' will be returned.
s utf-8c s' y � � SWn t k
r" t � SXd S( N( t
StopIterationt bytes( ( R+ ( s. /usr/lib64/python2.7/lib2to3/pgen2/tokenize.pyt read_or_stop s
c s� y | j d � } Wn t k
r' d SXt j | � } | sA d St | j d � � } y t | � } Wn! t k
r� t d | � � n X� r� | j
d k r� t d � � n | d 7} n | S( Nt asciii s unknown encoding: s utf-8s encoding problem: utf-8s -sig( t decodet UnicodeDecodeErrort Nonet cookie_ret matchRU R
R t LookupErrort SyntaxErrort name( R$ t line_stringR^ t encodingt codec( t bom_found( s. /usr/lib64/python2.7/lib2to3/pgen2/tokenize.pyt find_cookie s"
i s utf-8-sigN( RB R\ RR R RE t blank_reR^ ( R+ Rc t defaultRX Rf t firstt second( ( Re R+ s. /usr/lib64/python2.7/lib2to3/pgen2/tokenize.pyt detect_encoding� s0
c C s t � } | j | � S( s� Transform tokens back into Python source code.
Each element returned by the iterable must be a token sequence
with at least two elements, a token number and token value. If
only two tokens are passed, the resulting output is poor.
Round-trip invariant for full input:
Untokenized source will match input source exactly
Round-trip invariant for limited intput:
# Output text will tokenize the back to the input
t1 = [tok[:2] for tok in generate_tokens(f.readline)]
newcode = untokenize(t1)
readline = iter(newcode.splitlines(1)).next
t2 = [tok[:2] for tokin generate_tokens(readline)]
assert t1 == t2
( R. R ( R? t ut( ( s. /usr/lib64/python2.7/lib2to3/pgen2/tokenize.pyR F s c c sV d } } } t j d d } } d \ } } d } d g } x�y
| � }
Wn t k
rf d }
n X| d } d t |
� } } | r{|
s� t d |
f � n | j |
� } | r| j d � } } t | |
| |
| | f | |
f Vd \ } } d } q�| ra|
d d k ra|
d d
k rat | |
|
| t |
� f | f Vd } d } q@ q�| |
} | |
} q@ n`| d k r�| r�|
s�Pn d } xv | | k r|
| d k r�| d } n? |
| d k r�| t
d t
} n |
| d
k rd } n P| d } q�W| | k r'Pn |
| d k r|
| d k r�|
| j d � } | t | � } t | | | f | | t | � f |
f Vt
|
| | | f | t |
� f |
f Vq@ t
t f |
| d k |
| | | f | t |
� f |
f Vq@ n | | d k rI| j | � t |
| | d f | | f |
f Vn x� | | d k r�| | k r�t d d | | |
f � � n | d } t d | | f | | f |
f VqLWn$ |
s�t d | d f f � n d } x| | k r�t j |
| � } | r�| j d � \ } } | | f | | f | } } } |
| | !|
| } } | | k ss| d k r�| d k r�t | | | |
f Vq�| d k r�t } | d k r�t
} n | | | | |
f Vq�| d k r| j d � s�t � t | | | |
f Vq�| t k r�t | } | j |
| � } | rh| j d � } |
| | !} t | | | | f |
f Vq�| | f }
|
| } |
} Pq�| t k s�| d t k s�| d t k r(| d d k r| | f }
t | p�t | d p�t | d } |
| d } } |
} Pq�t | | | |
f Vq�| | k rKt | | | |
f Vq�| d k rzt
| | | | f |
f Vd } q�| d k r�| d } n | d k r�| d } n t | | | |
f Vq�t |
| | | f | | d f |
f V| d } q�Wq@ Wx2 | d D]& } t d | d f | d f d f VqWt d | d f | d f d f Vd S( sT
The generate_tokens() generator requires one argument, readline, which
must be a callable object which provides the same interface as the
readline() method of built-in file objects. Each call to the function
should return one line of input as a string. Alternately, readline
can be a callable function terminating with StopIteration:
readline = open(myfile).next # Example of alternate readline
The generator produces 5-tuples with these members: the token type; the
token string; a 2-tuple (srow, scol) of ints specifying the row and
column where the token begins in the source; a 2-tuple (erow, ecol) of
ints specifying the row and column where the token ends in the source;
and the line on which the token was found. The line passed is the
logical line; continuation lines are included.
i R t
0123456789R i s EOF in multi-line stringi����s \
i����s \
R4 s s s #
t #s
i����s3 unindent does not match any outer indentation levels
<tokenize>s EOF in multi-line statementt .s
i i s \s ([{s )]}N( R i ( R i ( t stringt
ascii_lettersR\ RV R; R R^ R# t STRINGt
ERRORTOKENt tabsizet rstript COMMENTR> R6 RF t IndentationErrorRG t
pseudoprogt spanRD R= t endswithR5 t
triple_quotedt endprogst
single_quotedRC t OPt ENDMARKER( R+ t lnumt parenlevt continuedt namecharst numcharst contstrt needcontt contlineRJ R$ t post maxt strstartt endprogt endmatchR# t columnt
comment_tokent nl_post pseudomatchR" t spost eposR t initialt newlinet indent( ( s. /usr/lib64/python2.7/lib2to3/pgen2/tokenize.pyR [ s�
&
$
#
'
'
$t __main__( s '''s """s r'''s r"""s R'''s R"""s u'''s u"""s U'''s U"""s b'''s b"""s B'''s B"""s ur'''s ur"""s Ur'''s Ur"""s uR'''s uR"""s UR'''s UR"""s br'''s br"""s Br'''s Br"""s bR'''s bR"""s BR'''s BR"""( R R s r's r"s R's R"s u's u"s U's U"s b's b"s B's B"s ur's ur"s Ur's Ur"s uR's uR"s UR's UR"s br's br"s Br's Br"s bR's bR"s BR's BR"( (R t __doc__t
__author__t __credits__Rp t ret codecsR R t lib2to3.pgen2.tokenR R t dirt xt __all__RW t NameErrort strR
R R t
Whitespacet Commentt Ignoret Namet Binnumbert Hexnumbert Octnumbert Decnumbert Intnumbert Exponentt
Pointfloatt Expfloatt Floatnumbert
Imagnumbert Numbert Singlet Doublet Single3t Double3t Triplet Stringt Operatort Brackett Specialt Funnyt
PlainTokent Tokent ContStrt PseudoExtrast PseudoTokent mapt compilet tokenprogRx t single3progt double3progR\ R| R{ R@ R} Rt t ExceptionR R R) R R* R. R] Rg RU Rk R R R t sysR; t argvt openR+ t stdin( ( ( s. /usr/lib64/python2.7/lib2to3/pgen2/tokenize.pyt <module> s�
/
'#
8 I � driver.pyc 0000644 00000014477 15047135540 0006575 0 ustar 00 �
{ fc @ s d Z d Z d d g Z d d l Z d d l Z d d l Z d d l Z d d l Z d d l Z d d l m
Z
m Z m Z m
Z
m Z d e f d � � YZ d � Z d
d e e d d � Z d � Z d
� Z d � Z e d k r e j e e � � � n d S( sZ Parser driver.
This provides a high-level interface to parse a file into a syntax tree.
s# Guido van Rossum <guido@python.org>t Drivert load_grammari����Ni ( t grammart parset tokent tokenizet pgenc B sV e Z d d d � Z e d � Z e d � Z e d � Z d e d � Z e d � Z RS( c C s: | | _ | d k r$ t j � } n | | _ | | _ d S( N( R t Nonet loggingt getLoggert loggert convert( t selfR R R
( ( s, /usr/lib64/python2.7/lib2to3/pgen2/driver.pyt __init__ s
c C s= t j | j | j � } | j � d } d } d } } } } }
d } x�| D]�} | \ } } } } }
| | | f k r
| | f | k s� t | | f | f � � | \ }
} | |
k r� | d |
| 7} |
} d } n | | k r
| |
| | !7} | } q
n | t j t j f k r`| | 7} | \ } } | j
d � rQ | d 7} d } qQ qQ n | t j k rt j
| } n | r�| j j d t j | | | � n | j | | | | f � r�| r�| j j d � n Pn d } | \ } } | j
d � rQ | d 7} d } qQ qQ Wt j d | | | | f � � | j S(
s4 Parse a series of tokens and return the syntax tree.i i u s
s %s %r (prefix=%r)s Stop.t s incomplete inputN( R t ParserR R t setupR t AssertionErrorR t COMMENTt NLt endswithR t OPt opmapR
t debugt tok_namet addtokent
ParseErrort rootnode( R t tokensR t pt linenot columnt typet valuet startt endt line_textt prefixt quintuplet s_linenot s_column( ( s, /usr/lib64/python2.7/lib2to3/pgen2/driver.pyt parse_tokens' sT
*
c C s" t j | j � } | j | | � S( s* Parse a stream and return the syntax tree.( R t generate_tokenst readlineR) ( R t streamR R ( ( s, /usr/lib64/python2.7/lib2to3/pgen2/driver.pyt parse_stream_rawW s c C s | j | | � S( s* Parse a stream and return the syntax tree.( R- ( R R, R ( ( s, /usr/lib64/python2.7/lib2to3/pgen2/driver.pyt parse_stream\ s c C s; t j | d | � } z | j | | � SWd | j � Xd S( s( Parse a file and return the syntax tree.t rN( t codecst openR. t close( R t filenamet encodingR R, ( ( s, /usr/lib64/python2.7/lib2to3/pgen2/driver.pyt
parse_file` s c C s+ t j t j | � j � } | j | | � S( s* Parse a string and return the syntax tree.( R R* t StringIOR+ R) ( R t textR R ( ( s, /usr/lib64/python2.7/lib2to3/pgen2/driver.pyt parse_stringh s N(
t __name__t
__module__R R
t FalseR) R- R. R5 R8 ( ( ( s, /usr/lib64/python2.7/lib2to3/pgen2/driver.pyR s 0 c C sR t j j | � \ } } | d k r- d } n | | d j t t t j � � d S( Ns .txtR t .s .pickle( t ost patht splitextt joint mapt strt syst version_info( t gtt headt tail( ( s, /usr/lib64/python2.7/lib2to3/pgen2/driver.pyt _generate_pickle_namen s s Grammar.txtc C s� | d k r t j � } n | d k r3 t | � n | } | sO t | | � r� | j d | � t j | � } | r� | j d | � y | j | � Wq� t k
r� } | j d | � q� Xq� n t
j � } | j | � | S( s' Load the grammar (maybe from a pickle).s! Generating grammar tables from %ss Writing grammar tables to %ss Writing failed: %sN(
R R R RH t _newert infoR t generate_grammart dumpt IOErrorR t Grammart load( RE t gpt savet forceR
t gt e( ( s, /usr/lib64/python2.7/lib2to3/pgen2/driver.pyR u s
c C sN t j j | � s t St j j | � s, t St j j | � t j j | � k S( s0 Inquire whether file a was written since file b.( R= R> t existsR; t Truet getmtime( t at b( ( s, /usr/lib64/python2.7/lib2to3/pgen2/driver.pyRI � s
c C sc t j j | � r t | � St t j j | � � } t j | | � } t j � } | j
| � | S( s� Normally, loads a pickled grammar by doing
pkgutil.get_data(package, pickled_grammar)
where *pickled_grammar* is computed from *grammar_source* by adding the
Python version and using a ``.pickle`` extension.
However, if *grammar_source* is an extant file, load_grammar(grammar_source)
is called instead. This facilitates using a packaged grammar file when needed
but preserves load_grammar's automatic regeneration behavior when possible.
( R= R> t isfileR RH t basenamet pkgutilt get_dataR RN t loads( t packaget grammar_sourcet pickled_namet dataRS ( ( s, /usr/lib64/python2.7/lib2to3/pgen2/driver.pyt load_packaged_grammar� s
c G sc | s t j d } n t j d t j d t j d d � x$ | D] } t | d t d t �q? Wt S( s� Main program, when run as a script: produce grammar pickle files.
Calls load_grammar for each argument, a path to a grammar text file.
i t levelR, t formats %(message)sRQ RR ( RC t argvR t basicConfigt INFOt stdoutR RV ( t argsRE ( ( s, /usr/lib64/python2.7/lib2to3/pgen2/driver.pyt main� s
t __main__( t __doc__t
__author__t __all__R0 R= R R\ R6 RC R R R R R R t objectR RH R RV R; R RI Rc Rk R9 t exitt int( ( ( s, /usr/lib64/python2.7/lib2to3/pgen2/driver.pyt <module> s$ (P
tokenize.pyo 0000644 00000040762 15047135540 0007142 0 ustar 00 �
{ fc @ s d Z d Z d Z d d l Z d d l Z d d l m Z m Z d d l Td d l m
Z
g e e
� D] Z e d d
k rg e ^ qg d d d
g Z
[
y e Wn e k
r� e Z n Xd � Z d � Z d � Z d Z d Z e e d e � e e � Z d Z d Z d Z d Z d Z e e e e e � Z d Z e d d � e e � Z d e Z e e e � Z e d e d � Z! e e! e e � Z" d Z# d Z$ d! Z% d" Z& e d# d$ � Z' e d% d& � Z( e d' d( d) d* d+ d, d- d. d/ � Z) d0 Z* e d1 d2 � Z+ e e) e* e+ � Z, e e" e, e( e � Z- e e- Z. e d3 e d4 d � d5 e d6 d � � Z/ e d e e' � Z0 e e e0 e" e, e/ e � Z1 e2 e j3 e. e1 e% e& f � \ Z4 Z5 Z6 Z7 i& e j3 e# � d4 6e j3 e$ � d6 6e6 d7 6e7 d8 6e6 d9 6e7 d: 6e6 d; 6e7 d< 6e6 d= 6e7 d> 6e6 d? 6e7 d@ 6e6 dA 6e7 dB 6e6 dC 6e7 dD 6e6 dE 6e7 dF 6e6 dG 6e7 dH 6e6 dI 6e7 dJ 6e6 dK 6e7 dL 6e6 dM 6e7 dN 6e6 dO 6e7 dP 6e6 dQ 6e7 dR 6e6 dS 6e7 dT 6d dU 6d dV 6d dW 6d dX 6d dY 6d dZ 6Z9 i Z: x d� D] Z; e; e: e; <q�Wi Z< x d� D] Z; e; e< e; <qWdw Z= dx e> f dy � � YZ? dz e> f d{ � � YZ@ d| � ZA eA d} � ZB d~ � ZC d d� d� � � YZD e j3 d� � ZE e j3 d� � ZF d� � ZG d� � ZH d� � ZI d� � ZJ eK d� k rd d lL ZL eM eL jN � d k reB eO eL jN d � jP � qeB eL jQ jP � n d S(� s� Tokenization help for Python programs.
generate_tokens(readline) is a generator that breaks a stream of
text into Python tokens. It accepts a readline-like method which is called
repeatedly to get the next line of input (or "" for EOF). It generates
5-tuples with these members:
the token type (see token.py)
the token (a string)
the starting (row, column) indices of the token (a 2-tuple of ints)
the ending (row, column) indices of the token (a 2-tuple of ints)
the original line (string)
It is designed to match the working of the Python tokenizer exactly, except
that it produces COMMENT tokens for comments and gives type OP for all
operators
Older entry points
tokenize_loop(readline, tokeneater)
tokenize(readline, tokeneater=printtoken)
are the same, except instead of generating tokens, tokeneater is a callback
function to which the 5 fields described above are passed as 5 arguments,
each time a new token is found.s Ka-Ping Yee <ping@lfw.org>s@ GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, Skip Montanaroi����N( t BOM_UTF8t lookup( t *i ( t tokeni t _t tokenizet generate_tokenst
untokenizec G s d d j | � d S( Nt (t |t )( t join( t choices( ( s. /usr/lib64/python2.7/lib2to3/pgen2/tokenize.pyt group0 t c G s t | � d S( NR ( R
( R ( ( s. /usr/lib64/python2.7/lib2to3/pgen2/tokenize.pyt any1 R c G s t | � d S( Nt ?( R
( R ( ( s. /usr/lib64/python2.7/lib2to3/pgen2/tokenize.pyt maybe2 R s [ \f\t]*s #[^\r\n]*s \\\r?\ns [a-zA-Z_]\w*s
0[bB][01]*s 0[xX][\da-fA-F]*[lL]?s 0[oO]?[0-7]*[lL]?s
[1-9]\d*[lL]?s [eE][-+]?\d+s \d+\.\d*s \.\d+s \d+s \d+[jJ]s [jJ]s [^'\\]*(?:\\.[^'\\]*)*'s [^"\\]*(?:\\.[^"\\]*)*"s% [^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''s% [^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""s [ubUB]?[rR]?'''s [ubUB]?[rR]?"""s& [uU]?[rR]?'[^\n'\\]*(?:\\.[^\n'\\]*)*'s& [uU]?[rR]?"[^\n"\\]*(?:\\.[^\n"\\]*)*"s \*\*=?s >>=?s <<=?s <>s !=s //=?s ->s [+\-*/%&@|^=<>]=?t ~s [][(){}]s \r?\ns [:;.,`@]s' [uUbB]?[rR]?'[^\n'\\]*(?:\\.[^\n'\\]*)*t 's' [uUbB]?[rR]?"[^\n"\\]*(?:\\.[^\n"\\]*)*t "s '''s """s r'''s r"""s u'''s u"""s b'''s b"""s ur'''s ur"""s br'''s br"""s R'''s R"""s U'''s U"""s B'''s B"""s uR'''s uR"""s Ur'''s Ur"""s UR'''s UR"""s bR'''s bR"""s Br'''s Br"""s BR'''s BR"""t rt Rt ut Ut bt Bs r's r"s R's R"s u's u"s U's U"s b's b"s B's B"s ur's ur"s Ur's Ur"s uR's uR"s UR's UR"s br's br"s Br's Br"s bR's bR"s BR's BR"i t
TokenErrorc B s e Z RS( ( t __name__t
__module__( ( ( s. /usr/lib64/python2.7/lib2to3/pgen2/tokenize.pyR � s t StopTokenizingc B s e Z RS( ( R R ( ( ( s. /usr/lib64/python2.7/lib2to3/pgen2/tokenize.pyR � s c C sA | \ } } | \ } } d | | | | t | t | � f GHd S( Ns %d,%d-%d,%d: %s %s( t tok_namet repr( t typeR t startt endt linet srowt scolt erowt ecol( ( s. /usr/lib64/python2.7/lib2to3/pgen2/tokenize.pyt
printtoken� s c C s) y t | | � Wn t k
r$ n Xd S( s:
The tokenize() function accepts two parameters: one representing the
input stream, and one providing an output mechanism for tokenize().
The first parameter, readline, must be a callable object which provides
the same interface as the readline() method of built-in file objects.
Each call to the function should return one line of input as a string.
The second parameter, tokeneater, must also be a callable object. It is
called once for each token, with five arguments, corresponding to the
tuples generated by generate_tokens().
N( t
tokenize_loopR ( t readlinet
tokeneater( ( s. /usr/lib64/python2.7/lib2to3/pgen2/tokenize.pyR � s
c C s% x t | � D] } | | � q
Wd S( N( R ( R+ R, t
token_info( ( s. /usr/lib64/python2.7/lib2to3/pgen2/tokenize.pyR* � s t Untokenizerc B s, e Z d � Z d � Z d � Z d � Z RS( c C s g | _ d | _ d | _ d S( Ni i ( t tokenst prev_rowt prev_col( t self( ( s. /usr/lib64/python2.7/lib2to3/pgen2/tokenize.pyt __init__� s c C s: | \ } } | | j } | r6 | j j d | � n d S( Nt ( R1 R/ t append( R2 R" t rowt colt
col_offset( ( s. /usr/lib64/python2.7/lib2to3/pgen2/tokenize.pyt add_whitespace� s
c C s� x� | D]� } t | � d k r3 | j | | � Pn | \ } } } } } | j | � | j j | � | \ | _ | _ | t t f k r | j d 7_ d | _ q q Wd j | j � S( Ni i i R (
t lent compatR9 R/ R5 R0 R1 t NEWLINEt NLR ( R2 t iterablet tt tok_typeR R" R# R$ ( ( s. /usr/lib64/python2.7/lib2to3/pgen2/tokenize.pyR � s
c C s% t } g } | j j } | \ } } | t t f k rC | d 7} n | t t f k r^ t } n x� | D]� } | d \ } } | t t f k r� | d 7} n | t k r� | j | � qe nZ | t k r� | j
� qe n> | t t f k r� t } n# | r| r| | d � t } n | | � qe Wd S( NR4 i i����( t FalseR/ R5 t NAMEt NUMBERR< R= t Truet INDENTt DEDENTt pop( R2 R R> t startlinet indentst toks_appendt toknumt tokvalt tok( ( s. /usr/lib64/python2.7/lib2to3/pgen2/tokenize.pyR; � s0
( R R R3 R9 R R; ( ( ( s. /usr/lib64/python2.7/lib2to3/pgen2/tokenize.pyR. � s s& ^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)s ^[ \t\f]*(?:[#\r\n]|$)c C s^ | d j � j d d � } | d k s7 | j d � r; d S| d k sV | j d
� rZ d S| S( s( Imitates get_normal_name in tokenizer.c.i R t -s utf-8s utf-8-s latin-1s
iso-8859-1s iso-latin-1s latin-1-s iso-8859-1-s iso-latin-1-( s latin-1s
iso-8859-1s iso-latin-1( s latin-1-s iso-8859-1-s iso-latin-1-( t lowert replacet
startswith( t orig_enct enc( ( s. /usr/lib64/python2.7/lib2to3/pgen2/tokenize.pyt _get_normal_name� s c s� t � d } d } � f d � } � f d � } | � } | j t � ra t � | d } d } n | sq | g f S| | � } | r� | | g f St j | � s� | | g f S| � } | s� | | g f S| | � } | r� | | | g f S| | | g f S( s
The detect_encoding() function is used to detect the encoding that should
be used to decode a Python source file. It requires one argument, readline,
in the same way as the tokenize() generator.
It will call readline a maximum of twice, and return the encoding used
(as a string) and a list of any lines (left as bytes) it has read
in.
It detects the encoding from the presence of a utf-8 bom or an encoding
cookie as specified in pep-0263. If both a bom and a cookie are present, but
disagree, a SyntaxError will be raised. If the encoding cookie is an invalid
charset, raise a SyntaxError. Note that if a utf-8 bom is found,
'utf-8-sig' is returned.
If no encoding is specified, then the default of 'utf-8' will be returned.
s utf-8c s' y � � SWn t k
r" t � SXd S( N( t
StopIterationt bytes( ( R+ ( s. /usr/lib64/python2.7/lib2to3/pgen2/tokenize.pyt read_or_stop s
c s� y | j d � } Wn t k
r' d SXt j | � } | sA d St | j d � � } y t | � } Wn! t k
r� t d | � � n X� r� | j
d k r� t d � � n | d 7} n | S( Nt asciii s unknown encoding: s utf-8s encoding problem: utf-8s -sig( t decodet UnicodeDecodeErrort Nonet cookie_ret matchRT R
R t LookupErrort SyntaxErrort name( R$ t line_stringR] t encodingt codec( t bom_found( s. /usr/lib64/python2.7/lib2to3/pgen2/tokenize.pyt find_cookie s"
i s utf-8-sigN( RA R[ RQ R RD t blank_reR] ( R+ Rb t defaultRW Re t firstt second( ( Rd R+ s. /usr/lib64/python2.7/lib2to3/pgen2/tokenize.pyt detect_encoding� s0
c C s t � } | j | � S( s� Transform tokens back into Python source code.
Each element returned by the iterable must be a token sequence
with at least two elements, a token number and token value. If
only two tokens are passed, the resulting output is poor.
Round-trip invariant for full input:
Untokenized source will match input source exactly
Round-trip invariant for limited intput:
# Output text will tokenize the back to the input
t1 = [tok[:2] for tok in generate_tokens(f.readline)]
newcode = untokenize(t1)
readline = iter(newcode.splitlines(1)).next
t2 = [tok[:2] for tokin generate_tokens(readline)]
assert t1 == t2
( R. R ( R> t ut( ( s. /usr/lib64/python2.7/lib2to3/pgen2/tokenize.pyR F s c c s@ d } } } t j d d } } d \ } } d } d g } x�y
| � }
Wn t k
rf d }
n X| d } d t |
� } } | r{|
s� t d |
f � n | j |
� } | r| j d � } } t | |
| |
| | f | |
f Vd \ } } d } q�| ra|
d d k ra|
d d
k rat | |
|
| t |
� f | f Vd } d } q@ q�| |
} | |
} q@ n`| d k r�| r�|
s�Pn d } xv | | k r|
| d k r�| d } n? |
| d k r�| t
d t
} n |
| d
k rd } n P| d } q�W| | k r'Pn |
| d k r|
| d k r�|
| j d � } | t | � } t | | | f | | t | � f |
f Vt
|
| | | f | t |
� f |
f Vq@ t
t f |
| d k |
| | | f | t |
� f |
f Vq@ n | | d k rI| j | � t |
| | d f | | f |
f Vn x� | | d k r�| | k r�t d d | | |
f � � n | d } t d | | f | | f |
f VqLWn$ |
s�t d | d f f � n d } x| | k r�t j |
| � } | r�| j d � \ } } | | f | | f | } } } |
| | !|
| } } | | k ss| d k r�| d k r�t | | | |
f Vq�| d k r�t } | d k r�t
} n | | | | |
f Vq�| d k r�t | | | |
f Vq�| t k rrt | } | j |
| � } | rR| j d � } |
| | !} t | | | | f |
f Vq�| | f }
|
| } |
} Pq�| t k s�| d t k s�| d t k r| d d k r�| | f }
t | p�t | d p�t | d } |
| d } } |
} Pq�t | | | |
f Vq�| | k r5t | | | |
f Vq�| d k rdt
| | | | f |
f Vd } q�| d k r}| d } n | d k r�| d } n t | | | |
f Vq�t |
| | | f | | d f |
f V| d } q�Wq@ Wx2 | d D]& } t d | d f | d f d f Vq�Wt d | d f | d f d f Vd S( sT
The generate_tokens() generator requires one argument, readline, which
must be a callable object which provides the same interface as the
readline() method of built-in file objects. Each call to the function
should return one line of input as a string. Alternately, readline
can be a callable function terminating with StopIteration:
readline = open(myfile).next # Example of alternate readline
The generator produces 5-tuples with these members: the token type; the
token string; a 2-tuple (srow, scol) of ints specifying the row and
column where the token begins in the source; a 2-tuple (erow, ecol) of
ints specifying the row and column where the token ends in the source;
and the line on which the token was found. The line passed is the
logical line; continuation lines are included.
i R t
0123456789R i s EOF in multi-line stringi����s \
i����s \
R4 s s s #
t #s
i����s3 unindent does not match any outer indentation levels
<tokenize>s EOF in multi-line statementt .i i s
s \s ([{s )]}N( R i ( R i ( t stringt
ascii_lettersR[ RU R: R R] R# t STRINGt
ERRORTOKENt tabsizet rstript COMMENTR= R5 RE t IndentationErrorRF t
pseudoprogt spanRC R< t
triple_quotedt endprogst
single_quotedRB t OPt ENDMARKER( R+ t lnumt parenlevt continuedt namecharst numcharst contstrt needcontt contlineRI R$ t post maxt strstartt endprogt endmatchR# t columnt
comment_tokent nl_post pseudomatchR" t spost eposR t initialt newlinet indent( ( s. /usr/lib64/python2.7/lib2to3/pgen2/tokenize.pyR [ s�
&
$
#
'
'
$t __main__( s '''s """s r'''s r"""s R'''s R"""s u'''s u"""s U'''s U"""s b'''s b"""s B'''s B"""s ur'''s ur"""s Ur'''s Ur"""s uR'''s uR"""s UR'''s UR"""s br'''s br"""s Br'''s Br"""s bR'''s bR"""s BR'''s BR"""( R R s r's r"s R's R"s u's u"s U's U"s b's b"s B's B"s ur's ur"s Ur's Ur"s uR's uR"s UR's UR"s br's br"s Br's Br"s bR's bR"s BR's BR"( (R t __doc__t
__author__t __credits__Ro t ret codecsR R t lib2to3.pgen2.tokenR R t dirt xt __all__RV t NameErrort strR
R R t
Whitespacet Commentt Ignoret Namet Binnumbert Hexnumbert Octnumbert Decnumbert Intnumbert Exponentt
Pointfloatt Expfloatt Floatnumbert
Imagnumbert Numbert Singlet Doublet Single3t Double3t Triplet Stringt Operatort Brackett Specialt Funnyt
PlainTokent Tokent ContStrt PseudoExtrast PseudoTokent mapt compilet tokenprogRw t single3progt double3progR[ Rz Ry R? R{ Rs t ExceptionR R R) R R* R. R\ Rf RT Rj R R R t sysR: t argvt openR+ t stdin( ( ( s. /usr/lib64/python2.7/lib2to3/pgen2/tokenize.pyt <module> s�
/
'#
8 I � token.pyo 0000644 00000004377 15047135541 0006435 0 ustar 00 �
{ fc @ s� d Z d Z d Z d Z d Z d Z d Z d Z d Z d Z d
Z
d Z d Z d
Z
d Z d Z d Z d Z d Z d Z d Z d Z d Z d Z d Z d Z d Z d Z d Z d Z d Z d Z d Z d! Z! d" Z" d# Z# d$ Z$ d% Z% d& Z& d' Z' d( Z( d) Z) d* Z* d+ Z+ d, Z, d- Z- d. Z. d/ Z/ d0 Z0 d1 Z1 d2 Z2 d3 Z3 d4 Z4 d5 Z5 d6 Z6 d7 Z7 d8 Z8 d9 Z9 d: Z: d; Z; i Z< xB e= � j> � D]1 \ Z? Z@ eA e@ � eA d � k r~e? e< e@ <q~q~Wd< � ZB d= � ZC d> � ZD d? S(@ s! Token constants (from "token.h").i i i i i i i i i i i
i i i
i i i i i i i i i i i i i i i i i i i i! i" i# i$ i% i& i' i( i) i* i+ i, i- i. i/ i0 i1 i2 i3 i4 i5 i6 i7 i8 i9 i c C s
| t k S( N( t NT_OFFSET( t x( ( s+ /usr/lib64/python2.7/lib2to3/pgen2/token.pyt
ISTERMINALL s c C s
| t k S( N( R ( R ( ( s+ /usr/lib64/python2.7/lib2to3/pgen2/token.pyt
ISNONTERMINALO s c C s
| t k S( N( t ENDMARKER( R ( ( s+ /usr/lib64/python2.7/lib2to3/pgen2/token.pyt ISEOFR s N(E t __doc__R t NAMEt NUMBERt STRINGt NEWLINEt INDENTt DEDENTt LPARt RPARt LSQBt RSQBt COLONt COMMAt SEMIt PLUSt MINUSt STARt SLASHt VBARt AMPERt LESSt GREATERt EQUALt DOTt PERCENTt BACKQUOTEt LBRACEt RBRACEt EQEQUALt NOTEQUALt LESSEQUALt GREATEREQUALt TILDEt
CIRCUMFLEXt LEFTSHIFTt
RIGHTSHIFTt
DOUBLESTARt PLUSEQUALt MINEQUALt STAREQUALt
SLASHEQUALt PERCENTEQUALt
AMPEREQUALt VBAREQUALt CIRCUMFLEXEQUALt LEFTSHIFTEQUALt RIGHTSHIFTEQUALt DOUBLESTAREQUALt DOUBLESLASHt DOUBLESLASHEQUALt ATt ATEQUALt OPt COMMENTt NLt RARROWt
ERRORTOKENt N_TOKENSR t tok_namet globalst itemst _namet _valuet typeR R R ( ( ( s+ /usr/lib64/python2.7/lib2to3/pgen2/token.pyt <module> s� token.pyc 0000644 00000004377 15047135541 0006421 0 ustar 00 �
{ fc @ s� d Z d Z d Z d Z d Z d Z d Z d Z d Z d Z d
Z
d Z d Z d
Z
d Z d Z d Z d Z d Z d Z d Z d Z d Z d Z d Z d Z d Z d Z d Z d Z d Z d Z d Z d! Z! d" Z" d# Z# d$ Z$ d% Z% d& Z& d' Z' d( Z( d) Z) d* Z* d+ Z+ d, Z, d- Z- d. Z. d/ Z/ d0 Z0 d1 Z1 d2 Z2 d3 Z3 d4 Z4 d5 Z5 d6 Z6 d7 Z7 d8 Z8 d9 Z9 d: Z: d; Z; i Z< xB e= � j> � D]1 \ Z? Z@ eA e@ � eA d � k r~e? e< e@ <q~q~Wd< � ZB d= � ZC d> � ZD d? S(@ s! Token constants (from "token.h").i i i i i i i i i i i
i i i
i i i i i i i i i i i i i i i i i i i i! i" i# i$ i% i& i' i( i) i* i+ i, i- i. i/ i0 i1 i2 i3 i4 i5 i6 i7 i8 i9 i c C s
| t k S( N( t NT_OFFSET( t x( ( s+ /usr/lib64/python2.7/lib2to3/pgen2/token.pyt
ISTERMINALL s c C s
| t k S( N( R ( R ( ( s+ /usr/lib64/python2.7/lib2to3/pgen2/token.pyt
ISNONTERMINALO s c C s
| t k S( N( t ENDMARKER( R ( ( s+ /usr/lib64/python2.7/lib2to3/pgen2/token.pyt ISEOFR s N(E t __doc__R t NAMEt NUMBERt STRINGt NEWLINEt INDENTt DEDENTt LPARt RPARt LSQBt RSQBt COLONt COMMAt SEMIt PLUSt MINUSt STARt SLASHt VBARt AMPERt LESSt GREATERt EQUALt DOTt PERCENTt BACKQUOTEt LBRACEt RBRACEt EQEQUALt NOTEQUALt LESSEQUALt GREATEREQUALt TILDEt
CIRCUMFLEXt LEFTSHIFTt
RIGHTSHIFTt
DOUBLESTARt PLUSEQUALt MINEQUALt STAREQUALt
SLASHEQUALt PERCENTEQUALt
AMPEREQUALt VBAREQUALt CIRCUMFLEXEQUALt LEFTSHIFTEQUALt RIGHTSHIFTEQUALt DOUBLESTAREQUALt DOUBLESLASHt DOUBLESLASHEQUALt ATt ATEQUALt OPt COMMENTt NLt RARROWt
ERRORTOKENt N_TOKENSR t tok_namet globalst itemst _namet _valuet typeR R R ( ( ( s+ /usr/lib64/python2.7/lib2to3/pgen2/token.pyt <module> s� literals.pyo 0000644 00000003372 15047135541 0007126 0 ustar 00 �
{ fc @ s� d Z d d l Z i
d d 6d d 6d d 6d d
6d d 6d
d 6d d 6d d 6d d 6d d 6Z d � Z d � Z d � Z e d k r� e � n d S( s< Safely evaluate Python string literals without using eval().i����Ns t as t bs t fs
t ns
t rs t ts t vt 't "s \c C s� | j d d � \ } } t j | � } | d k r7 | S| j d � r� | d } t | � d k ru t d | � � n y t | d � } Wq� t k
r� t d | � � q� Xn7 y t | d � } Wn! t k
r� t d | � � n Xt | � S( Ni i t xi s! invalid hex string escape ('\%s')i i s# invalid octal string escape ('\%s')( t groupt simple_escapest gett Nonet
startswitht lent
ValueErrort intt chr( t mt allt tailt esct hexest i( ( s. /usr/lib64/python2.7/lib2to3/pgen2/literals.pyt escape s"
c C sX | d } | d | d k r+ | d } n | t | � t | � !} t j d t | � S( Ni i s) \\(\'|\"|\\|[abfnrtv]|x.{0,2}|[0-7]{1,3})( R t ret subR ( t st q( ( s. /usr/lib64/python2.7/lib2to3/pgen2/literals.pyt
evalString( s
c C s_ xX t d � D]J } t | � } t | � } t | � } | | k r
| G| G| G| GHq
q
Wd S( Ni ( t rangeR t reprR ( R t cR t e( ( s. /usr/lib64/python2.7/lib2to3/pgen2/literals.pyt test2 s t __main__( t __doc__R R R R R# t __name__( ( ( s. /usr/lib64/python2.7/lib2to3/pgen2/literals.pyt <module> s
conv.pyc 0000644 00000017775 15047135541 0006254 0 ustar 00 �
{ fc @ sE d Z d d l Z d d l m Z m Z d e j f d � � YZ d S( s� Convert graminit.[ch] spit out by pgen to Python code.
Pgen is the Python parser generator. It is useful to quickly create a
parser from a grammar file in Python's grammar notation. But I don't
want my parsers to be written in C (yet), so I'm translating the
parsing tables to Python data structures and writing a Python parse
engine.
Note that the token numbers are constants determined by the standard
Python tokenizer. The standard token module defines these numbers and
their names (the names are not used much). The token numbers are
hardcoded into the Python tokenizer and into pgen. A Python
implementation of the Python tokenizer is also available, in the
standard tokenize module.
On the other hand, symbol numbers (representing the grammar's
non-terminals) are assigned by pgen based on the actual grammar
input.
Note: this module is pretty much obsolete; the pgen module generates
equivalent grammar tables directly from the Grammar.txt input file
without having to invoke the Python pgen C program.
i����N( t grammart tokent Converterc B s2 e Z d Z d � Z d � Z d � Z d � Z RS( s2 Grammar subclass that reads classic pgen output files.
The run() method reads the tables as produced by the pgen parser
generator, typically contained in two C files, graminit.h and
graminit.c. The other methods are for internal use only.
See the base class for more documentation.
c C s( | j | � | j | � | j � d S( s< Load the grammar tables from the text files written by pgen.N( t parse_graminit_ht parse_graminit_ct
finish_off( t selft
graminit_ht
graminit_c( ( s* /usr/lib64/python2.7/lib2to3/pgen2/conv.pyt run/ s
c C s y t | � } Wn# t k
r5 } d | | f GHt SXi | _ i | _ d } x� | D]� } | d 7} t j d | � } | r� | j � r� d | | | j � f GHqU | j � \ } } t | � } | | j k s� t
� | | j k s� t
� | | j | <| | j | <qU Wt S( s� Parse the .h file written by pgen. (Internal)
This file is a sequence of #define statements defining the
nonterminals of the grammar as numbers. We build two tables
mapping the numbers to names and back.
s Can't open %s: %si i s ^#define\s+(\w+)\s+(\d+)$s %s(%s): can't parse %s( t opent IOErrort Falset
symbol2numbert
number2symbolt ret matcht stript groupst intt AssertionErrort True( R t filenamet ft errt linenot linet mot symbolt number( ( s* /usr/lib64/python2.7/lib2to3/pgen2/conv.pyR 5 s*
c ! C s� y t | � } Wn# t k
r5 } d | | f GHt SXd } | d | j � } } | d k sq t | | f � � | d | j � } } | d k s� t | | f � � | d | j � } } i } g } x�| j d � r�xC| j d � r t j d | � } | st | | f � � t t | j
� � \ } }
} g } x� t | � D]u }
| d | j � } } t j d | � } | s�t | | f � � t t | j
� � \ } } | j | | f � qHW| d | j � } } | d k s�t | | f � � | | | |
f <| d | j � } } q� Wt j d
| � } | sKt | | f � � t t | j
� � \ } } | t
| � k s�t | | f � � g } x� t | � D]� }
| d | j � } } t j d | � } | s�t | | f � � t t | j
� � \ } } }
| | |
f } | t
| � k s6t | | f � � | j | � q�W| j | � | d | j � } } | d k s�t | | f � � | d | j � } } q� W| | _ i } t j d | � } | s�t | | f � � t | j d � � } x�t | � D]�} | d | j � } } t j d
| � } | sFt | | f � � | j d � } t t | j d d d d � � \ } } } } | j | | k s�t | | f � � | j | | k s�t | | f � � | d k s�t | | f � � | | } | t
| � k st | | f � � | d | j � } } t j d | � } | sYt | | f � � i } t | j d � � } x` t | � D]R \ } } t | � } x7 t d � D]) } | d | >@r�d | | d | <q�q�Wq�W| | f | | <q�W| d | j � } } | d k s t | | f � � | | _ g } | d | j � } } t j d | � } | spt | | f � � t | j d � � } x� t | � D]� } | d | j � } } t j d | � } | s�t | | f � � | j
� \ } } t | � } | d k r d } n t | � } | j | | f � q�W| d | j � } } | d k sdt | | f � � | | _ | d | j � } } | d k s�t | | f � � | d | j � } } t j d | � } | s�t | | f � � t | j d � � } | t
| j � k st � | d | j � } } | d k sHt | | f � � | d | j � } } t j d | � } | s�t | | f � � t | j d � � } | t
| j � k s�t | | f � � | d | j � } } t j d | � } | s t | | f � � t | j d � � } | | j k s< t | | f � � | | _ | d | j � } } | d k sz t | | f � � y | d | j � } } Wn t k
r� n Xd s� t | | f � � d S( s� Parse the .c file written by pgen. (Internal)
The file looks as follows. The first two lines are always this:
#include "pgenheaders.h"
#include "grammar.h"
After that come four blocks:
1) one or more state definitions
2) a table defining dfas
3) a table defining labels
4) a struct defining the grammar
A state definition has the following form:
- one or more arc arrays, each of the form:
static arc arcs_<n>_<m>[<k>] = {
{<i>, <j>},
...
};
- followed by a state array, of the form:
static state states_<s>[<t>] = {
{<k>, arcs_<n>_<m>},
...
};
s Can't open %s: %si i s #include "pgenheaders.h"
s #include "grammar.h"
s static arc s) static arc arcs_(\d+)_(\d+)\[(\d+)\] = {$s \s+{(\d+), (\d+)},$s };
s' static state states_(\d+)\[(\d+)\] = {$s \s+{(\d+), arcs_(\d+)_(\d+)},$s static dfa dfas\[(\d+)\] = {$s0 \s+{(\d+), "(\w+)", (\d+), (\d+), states_(\d+),$i i i i s \s+("(?:\\\d\d\d)*")},$i s! static label labels\[(\d+)\] = {$s \s+{(\d+), (0|"\w+")},$t 0s grammar _PyParser_Grammar = {
s
\s+(\d+),$s dfas,
s \s+{(\d+), labels},$s \s+(\d+)$N( R
R R t nextR t
startswithR R t mapR R t ranget appendt lent statest groupR
R t evalt enumeratet ordt dfast Nonet labelst startt
StopIteration(! R R R R R R t allarcsR% R t nt mt kt arcst _t it jt st tt stateR* t ndfasR R t xt yt zt firstt rawbitsett ct byteR, t nlabelsR- ( ( s* /usr/lib64/python2.7/lib2to3/pgen2/conv.pyR T s� $ $
-%%
$ '!
c C s� i | _ i | _ xj t | j � D]Y \ } \ } } | t j k r_ | d k r_ | | j | <q" | d k r" | | j | <q" q" Wd S( s1 Create additional useful structures. (Internal).N( t keywordst tokensR( R, R t NAMER+ ( R t ilabelt typet value( ( s* /usr/lib64/python2.7/lib2to3/pgen2/conv.pyR � s " ( t __name__t
__module__t __doc__R R R R ( ( ( s* /usr/lib64/python2.7/lib2to3/pgen2/conv.pyR $ s
�( RK R t pgen2R R t GrammarR ( ( ( s* /usr/lib64/python2.7/lib2to3/pgen2/conv.pyt <module> s parse.pyo 0000644 00000016035 15047135541 0006421 0 ustar 00 �
{ fc @ sF d Z d d l m Z d e f d � � YZ d e f d � � YZ d S( s� Parser engine for the grammar tables generated by pgen.
The grammar table must be loaded first.
See Parser/parser.c in the Python distribution for additional info on
how this parsing engine works.
i ( t tokent
ParseErrorc B s e Z d Z d � Z RS( s( Exception to signal the parser is stuck.c C sH t j | d | | | | f � | | _ | | _ | | _ | | _ d S( Ns! %s: type=%r, value=%r, context=%r( t Exceptiont __init__t msgt typet valuet context( t selfR R R R ( ( s+ /usr/lib64/python2.7/lib2to3/pgen2/parse.pyR s ( t __name__t
__module__t __doc__R ( ( ( s+ /usr/lib64/python2.7/lib2to3/pgen2/parse.pyR s t Parserc B sS e Z d Z d d � Z d d � Z d � Z d � Z d � Z d � Z d � Z
RS( s5 Parser engine.
The proper usage sequence is:
p = Parser(grammar, [converter]) # create instance
p.setup([start]) # prepare for parsing
<for each input token>:
if p.addtoken(...): # parse a token; may raise ParseError
break
root = p.rootnode # root of abstract syntax tree
A Parser instance may be reused by calling setup() repeatedly.
A Parser instance contains state pertaining to the current token
sequence, and should not be used concurrently by different threads
to parse separate token sequences.
See driver.py for how to get input tokens by tokenizing a file or
string.
Parsing is complete when addtoken() returns True; the root of the
abstract syntax tree can then be retrieved from the rootnode
instance variable. When a syntax error occurs, addtoken() raises
the ParseError exception. There is no error recovery; the parser
cannot be used after a syntax error was reported (but it can be
reinitialized by calling setup()).
c C s | | _ | p d � | _ d S( s� Constructor.
The grammar argument is a grammar.Grammar instance; see the
grammar module for more information.
The parser is not ready yet for parsing; you must call the
setup() method to get it started.
The optional convert argument is a function mapping concrete
syntax tree nodes to abstract syntax tree nodes. If not
given, no conversion is done and the syntax tree produced is
the concrete syntax tree. If given, it must be a function of
two arguments, the first being the grammar (a grammar.Grammar
instance), and the second being the concrete syntax tree node
to be converted. The syntax tree is converted from the bottom
up.
A concrete syntax tree node is a (type, value, context, nodes)
tuple, where type is the node type (a token or symbol number),
value is None for symbols and a string for tokens, context is
None or an opaque value used for error reporting (typically a
(lineno, offset) pair), and nodes is a list of children for
symbols, and None for tokens.
An abstract syntax tree node may be anything; this is entirely
up to the converter function.
c S s | S( N( ( t grammart node( ( s+ /usr/lib64/python2.7/lib2to3/pgen2/parse.pyt <lambda>W t N( R
t convert( R R
R ( ( s+ /usr/lib64/python2.7/lib2to3/pgen2/parse.pyR 9 s c C sk | d k r | j j } n | d d g f } | j j | d | f } | g | _ d | _ t � | _ d S( s� Prepare for parsing.
This *must* be called before starting to parse.
The optional argument is an alternative start symbol; it
defaults to the grammar's start symbol.
You can use a Parser instance to parse any number of programs;
each time you call setup() the parser is reset to an initial
state determined by the (implicit or explicit) start symbol.
i N( t NoneR
t startt dfast stackt rootnodet sett
used_names( R R t newnodet
stackentry( ( s+ /usr/lib64/python2.7/lib2to3/pgen2/parse.pyt setupY s
c C s� | j | | | � } x�t r�| j d \ } } } | \ } } | | }
x_|
D]\ } } | j j | \ }
} | | k r� | j | | | | � | } xV | | d | f g k r� | j � | j s� t S| j d \ } } } | \ } } q� Wt S|
d k rQ | j j |
} | \ } } | | k rS| j |
| j j |
| | � PqSqQ qQ Wd | f |
k r�| j � | j s�t
d | | | � � q�q t
d | | | � � q Wd S( s< Add a token; return True iff this is the end of the program.i����i i s too much inputs bad inputN( t classifyt TrueR R
t labelst shiftt popt FalseR t pushR ( R R R R t ilabelt dfat stateR t statest firstt arcst it newstatet tt vt itsdfat itsstatest itsfirst( ( s+ /usr/lib64/python2.7/lib2to3/pgen2/parse.pyt addtokenq s:
c C s� | t j k rG | j j | � | j j j | � } | d k rG | Sn | j j j | � } | d k r� t d | | | � � n | S( s& Turn a token into a label. (Internal)s bad tokenN(
R t NAMER t addR
t keywordst getR t tokensR ( R R R R R# ( ( s+ /usr/lib64/python2.7/lib2to3/pgen2/parse.pyR � s c C sw | j d \ } } } | | | d f } | j | j | � } | d k r] | d j | � n | | | f | j d <d S( s Shift a token. (Internal)i����N( R R R R
t append( R R R R* R R$ R% R R ( ( s+ /usr/lib64/python2.7/lib2to3/pgen2/parse.pyR � s c C s[ | j d \ } } } | d | g f } | | | f | j d <| j j | d | f � d S( s Push a nonterminal. (Internal)i����i N( R R R6 ( R R t newdfaR* R R$ R% R R ( ( s+ /usr/lib64/python2.7/lib2to3/pgen2/parse.pyR" � s c C s� | j j � \ } } } | j | j | � } | d k r� | j rl | j d \ } } } | d j | � q� | | _ | j | j _ n d S( s Pop a nonterminal. (Internal)i����N( R R R R
R R6 R R ( R t popdfat popstatet popnodeR R$ R% R ( ( s+ /usr/lib64/python2.7/lib2to3/pgen2/parse.pyR � s N( R R
R R R R R0 R R R" R ( ( ( s+ /usr/lib64/python2.7/lib2to3/pgen2/parse.pyR s 0 N( R R R R R t objectR ( ( ( s+ /usr/lib64/python2.7/lib2to3/pgen2/parse.pyt <module> s grammar.pyo 0000644 00000017004 15047135541 0006732 0 ustar 00 �
{ fc @ s� d Z d d l Z d d l Z d d l m Z m Z d e f d � � YZ d � Z d Z i Z
xB e j � D]4 Z e rl e j
� \ Z Z e e e � e
e <ql ql Wd S( s� This module defines the data structures used to represent a grammar.
These are a bit arcane because they are derived from the data
structures used by Python's 'pgen' parser generator.
There's also a table here mapping operators to their names in the
token module; the Python tokenize module reports all operators as the
fallback token code OP, but the parser needs the actual token code.
i����Ni ( t tokent tokenizet Grammarc B sD e Z d Z d � Z d � Z d � Z d � Z d � Z d � Z RS( s� Pgen parsing tables conversion class.
Once initialized, this class supplies the grammar tables for the
parsing engine implemented by parse.py. The parsing engine
accesses the instance variables directly. The class here does not
provide initialization of the tables; several subclasses exist to
do this (see the conv and pgen modules).
The load() method reads the tables from a pickle file, which is
much faster than the other ways offered by subclasses. The pickle
file is written by calling dump() (after loading the grammar
tables using a subclass). The report() method prints a readable
representation of the tables to stdout, for debugging.
The instance variables are as follows:
symbol2number -- a dict mapping symbol names to numbers. Symbol
numbers are always 256 or higher, to distinguish
them from token numbers, which are between 0 and
255 (inclusive).
number2symbol -- a dict mapping numbers to symbol names;
these two are each other's inverse.
states -- a list of DFAs, where each DFA is a list of
states, each state is a list of arcs, and each
arc is a (i, j) pair where i is a label and j is
a state number. The DFA number is the index into
this list. (This name is slightly confusing.)
Final states are represented by a special arc of
the form (0, j) where j is its own state number.
dfas -- a dict mapping symbol numbers to (DFA, first)
pairs, where DFA is an item from the states list
above, and first is a set of tokens that can
begin this grammar rule (represented by a dict
whose values are always 1).
labels -- a list of (x, y) pairs where x is either a token
number or a symbol number, and y is either None
or a string; the strings are keywords. The label
number is the index in this list; label numbers
are used to mark state transitions (arcs) in the
DFAs.
start -- the number of the grammar's start symbol.
keywords -- a dict mapping keyword strings to arc labels.
tokens -- a dict mapping token numbers to arc labels.
c C sX i | _ i | _ g | _ i | _ d g | _ i | _ i | _ i | _ d | _ d S( Ni t EMPTYi ( i R ( t
symbol2numbert
number2symbolt statest dfast labelst keywordst tokenst symbol2labelt start( t self( ( s- /usr/lib64/python2.7/lib2to3/pgen2/grammar.pyt __init__M s c C s> t | d � �) } t | j � } t j | | d � Wd QXd S( s� Dump the grammar tables to a pickle file.
dump() recursively changes all dict to OrderedDict, so the pickled file
is not exactly the same as what was passed in to dump(). load() uses the
pickled file to create the tables, but only changes OrderedDict to dict
at the top level; it does not recursively change OrderedDict to dict.
So, the loaded tables are different from the original tables that were
passed to load() in that some of the OrderedDict (from the pickled file)
are not changed back to dict. For parsing, this has no effect on
performance because OrderedDict uses dict's __getitem__ with nothing in
between.
t wbi N( t opent _make_deterministict __dict__t picklet dump( R
t filenamet ft d( ( s- /usr/lib64/python2.7/lib2to3/pgen2/grammar.pyR X s
c C s< t | d � } t j | � } | j � | j j | � d S( s+ Load the grammar tables from a pickle file.t rbN( R R t loadt closeR t update( R
R R R ( ( s- /usr/lib64/python2.7/lib2to3/pgen2/grammar.pyR i s
c C s | j j t j | � � d S( s3 Load the grammar tables from a pickle bytes object.N( R R R t loads( R
t pkl( ( s- /usr/lib64/python2.7/lib2to3/pgen2/grammar.pyR p s c C sf | j � } x- d D]% } t | | t | | � j � � q W| j | _ | j | _ | j | _ | S( s#
Copy the grammar.
R R R R R
R ( R R R R R
R ( t __class__t setattrt getattrt copyR R R ( R
t newt dict_attr( ( s- /usr/lib64/python2.7/lib2to3/pgen2/grammar.pyR! t s
#
c C sz d d l m } d GH| | j � d GH| | j � d GH| | j � d GH| | j � d GH| | j � d G| j GHd S(
s: Dump the grammar tables to standard output, for debugging.i����( t pprintt s2nt n2sR R R R N( R$ R R R R R R ( R
R$ ( ( s- /usr/lib64/python2.7/lib2to3/pgen2/grammar.pyt report� s
( t __name__t
__module__t __doc__R R R R R! R' ( ( ( s- /usr/lib64/python2.7/lib2to3/pgen2/grammar.pyR s 4
c C s� t | t � r2 t j t d � | j � D� � � St | t � r^ g | D] } t | � ^ qH St | t � r� t d � | D� � S| S( Nc s s' | ] \ } } | t | � f Vq d S( N( R ( t .0t kt v( ( s- /usr/lib64/python2.7/lib2to3/pgen2/grammar.pys <genexpr>� s c s s | ] } t | � Vq d S( N( R ( R+ t e( ( s- /usr/lib64/python2.7/lib2to3/pgen2/grammar.pys <genexpr>� s ( t
isinstancet dictt collectionst OrderedDictt sortedt iteritemst listR t tuple( t topR. ( ( s- /usr/lib64/python2.7/lib2to3/pgen2/grammar.pyR � s s
( LPAR
) RPAR
[ LSQB
] RSQB
: COLON
, COMMA
; SEMI
+ PLUS
- MINUS
* STAR
/ SLASH
| VBAR
& AMPER
< LESS
> GREATER
= EQUAL
. DOT
% PERCENT
` BACKQUOTE
{ LBRACE
} RBRACE
@ AT
@= ATEQUAL
== EQEQUAL
!= NOTEQUAL
<> NOTEQUAL
<= LESSEQUAL
>= GREATEREQUAL
~ TILDE
^ CIRCUMFLEX
<< LEFTSHIFT
>> RIGHTSHIFT
** DOUBLESTAR
+= PLUSEQUAL
-= MINEQUAL
*= STAREQUAL
/= SLASHEQUAL
%= PERCENTEQUAL
&= AMPEREQUAL
|= VBAREQUAL
^= CIRCUMFLEXEQUAL
<<= LEFTSHIFTEQUAL
>>= RIGHTSHIFTEQUAL
**= DOUBLESTAREQUAL
// DOUBLESLASH
//= DOUBLESLASHEQUAL
-> RARROW
( R* R1 R t R R t objectR R t opmap_rawt opmapt
splitlinest linet splitt opt nameR ( ( ( s- /usr/lib64/python2.7/lib2to3/pgen2/grammar.pyt <module>
s z = literals.pyc 0000644 00000003742 15047135541 0007113 0 ustar 00 �
{ fc @ s� d Z d d l Z i
d d 6d d 6d d 6d d
6d d 6d
d 6d d 6d d 6d d 6d d 6Z d � Z d � Z d � Z e d k r� e � n d S( s< Safely evaluate Python string literals without using eval().i����Ns t as t bs t fs
t ns
t rs t ts t vt 't "s \c C s | j d d � \ } } | j d � s- t � t j | � } | d k rL | S| j d � r� | d } t | � d k r� t d | � � n y t | d � } Wq� t k
r� t d | � � q� Xn7 y t | d � } Wn! t k
r� t d | � � n Xt | � S(
Ni i s \t xi s! invalid hex string escape ('\%s')i i s# invalid octal string escape ('\%s')(
t groupt
startswitht AssertionErrort simple_escapest gett Nonet lent
ValueErrort intt chr( t mt allt tailt esct hexest i( ( s. /usr/lib64/python2.7/lib2to3/pgen2/literals.pyt escape s$
c C s� | j d � s4 | j d � s4 t t | d � � � | d } | d | d k r_ | d } n | j | � s� t t | t | � � � � t | � d t | � k s� t � | t | � t | � !} t j d t | � S( NR R i i i i s) \\(\'|\"|\\|[abfnrtv]|x.{0,2}|[0-7]{1,3})( R R t reprt endswithR t ret subR ( t st q( ( s. /usr/lib64/python2.7/lib2to3/pgen2/literals.pyt
evalString( s 4
,"c C s_ xX t d � D]J } t | � } t | � } t | � } | | k r
| G| G| G| GHq
q
Wd S( Ni ( t rangeR R R! ( R t cR t e( ( s. /usr/lib64/python2.7/lib2to3/pgen2/literals.pyt test2 s t __main__( t __doc__R R
R R! R% t __name__( ( ( s. /usr/lib64/python2.7/lib2to3/pgen2/literals.pyt <module> s
__init__.pyc 0000644 00000000256 15047135541 0007030 0 ustar 00 �
{ fc @ s
d Z d S( s The pgen2 package.N( t __doc__( ( ( s. /usr/lib64/python2.7/lib2to3/pgen2/__init__.pyt <module> t driver.pyo 0000644 00000014400 15047135541 0006574 0 ustar 00 �
{ fc @ s d Z d Z d d g Z d d l Z d d l Z d d l Z d d l Z d d l Z d d l Z d d l m
Z
m Z m Z m
Z
m Z d e f d � � YZ d � Z d
d e e d d � Z d � Z d
� Z d � Z e d k r e j e e � � � n d S( sZ Parser driver.
This provides a high-level interface to parse a file into a syntax tree.
s# Guido van Rossum <guido@python.org>t Drivert load_grammari����Ni ( t grammart parset tokent tokenizet pgenc B sV e Z d d d � Z e d � Z e d � Z e d � Z d e d � Z e d � Z RS( c C s: | | _ | d k r$ t j � } n | | _ | | _ d S( N( R t Nonet loggingt getLoggert loggert convert( t selfR R R
( ( s, /usr/lib64/python2.7/lib2to3/pgen2/driver.pyt __init__ s
c C s t j | j | j � } | j � d } d } d } } } } }
d } x�| D]�} | \ } } } } }
| | | f k r� | \ }
} | |
k r� | d |
| 7} |
} d } n | | k r� | |
| | !7} | } q� n | t j t j f k r6| | 7} | \ } } | j d � rQ | d 7} d } qQ qQ n | t
j k rUt j | } n | r~| j
j d t
j | | | � n | j | | | | f � r�| r�| j
j d � n Pn d } | \ } } | j d � rQ | d 7} d } qQ qQ Wt j d | | | | f � � | j S(
s4 Parse a series of tokens and return the syntax tree.i i u s
s %s %r (prefix=%r)s Stop.t s incomplete inputN( R t ParserR R t setupR R t COMMENTt NLt endswithR t OPt opmapR
t debugt tok_namet addtokent
ParseErrort rootnode( R t tokensR t pt linenot columnt typet valuet startt endt line_textt prefixt quintuplet s_linenot s_column( ( s, /usr/lib64/python2.7/lib2to3/pgen2/driver.pyt parse_tokens' sR
c C s" t j | j � } | j | | � S( s* Parse a stream and return the syntax tree.( R t generate_tokenst readlineR( ( R t streamR R ( ( s, /usr/lib64/python2.7/lib2to3/pgen2/driver.pyt parse_stream_rawW s c C s | j | | � S( s* Parse a stream and return the syntax tree.( R, ( R R+ R ( ( s, /usr/lib64/python2.7/lib2to3/pgen2/driver.pyt parse_stream\ s c C s; t j | d | � } z | j | | � SWd | j � Xd S( s( Parse a file and return the syntax tree.t rN( t codecst openR- t close( R t filenamet encodingR R+ ( ( s, /usr/lib64/python2.7/lib2to3/pgen2/driver.pyt
parse_file` s c C s+ t j t j | � j � } | j | | � S( s* Parse a string and return the syntax tree.( R R) t StringIOR* R( ( R t textR R ( ( s, /usr/lib64/python2.7/lib2to3/pgen2/driver.pyt parse_stringh s N(
t __name__t
__module__R R
t FalseR( R, R- R4 R7 ( ( ( s, /usr/lib64/python2.7/lib2to3/pgen2/driver.pyR s 0 c C sR t j j | � \ } } | d k r- d } n | | d j t t t j � � d S( Ns .txtR t .s .pickle( t ost patht splitextt joint mapt strt syst version_info( t gtt headt tail( ( s, /usr/lib64/python2.7/lib2to3/pgen2/driver.pyt _generate_pickle_namen s s Grammar.txtc C s� | d k r t j � } n | d k r3 t | � n | } | sO t | | � r� | j d | � t j | � } | r� | j d | � y | j | � Wq� t k
r� } | j d | � q� Xq� n t
j � } | j | � | S( s' Load the grammar (maybe from a pickle).s! Generating grammar tables from %ss Writing grammar tables to %ss Writing failed: %sN(
R R R RG t _newert infoR t generate_grammart dumpt IOErrorR t Grammart load( RD t gpt savet forceR
t gt e( ( s, /usr/lib64/python2.7/lib2to3/pgen2/driver.pyR u s
c C sN t j j | � s t St j j | � s, t St j j | � t j j | � k S( s0 Inquire whether file a was written since file b.( R< R= t existsR: t Truet getmtime( t at b( ( s, /usr/lib64/python2.7/lib2to3/pgen2/driver.pyRH � s
c C sc t j j | � r t | � St t j j | � � } t j | | � } t j � } | j
| � | S( s� Normally, loads a pickled grammar by doing
pkgutil.get_data(package, pickled_grammar)
where *pickled_grammar* is computed from *grammar_source* by adding the
Python version and using a ``.pickle`` extension.
However, if *grammar_source* is an extant file, load_grammar(grammar_source)
is called instead. This facilitates using a packaged grammar file when needed
but preserves load_grammar's automatic regeneration behavior when possible.
( R< R= t isfileR RG t basenamet pkgutilt get_dataR RM t loads( t packaget grammar_sourcet pickled_namet dataRR ( ( s, /usr/lib64/python2.7/lib2to3/pgen2/driver.pyt load_packaged_grammar� s
c G sc | s t j d } n t j d t j d t j d d � x$ | D] } t | d t d t �q? Wt S( s� Main program, when run as a script: produce grammar pickle files.
Calls load_grammar for each argument, a path to a grammar text file.
i t levelR+ t formats %(message)sRP RQ ( RB t argvR t basicConfigt INFOt stdoutR RU ( t argsRD ( ( s, /usr/lib64/python2.7/lib2to3/pgen2/driver.pyt main� s
t __main__( t __doc__t
__author__t __all__R/ R< R R[ R5 RB R R R R R R t objectR RG R RU R: R RH Rb Rj R8 t exitt int( ( ( s, /usr/lib64/python2.7/lib2to3/pgen2/driver.pyt <module> s$ (P
grammar.pyc 0000644 00000017004 15047135541 0006716 0 ustar 00 �
{ fc @ s� d Z d d l Z d d l Z d d l m Z m Z d e f d � � YZ d � Z d Z i Z
xB e j � D]4 Z e rl e j
� \ Z Z e e e � e
e <ql ql Wd S( s� This module defines the data structures used to represent a grammar.
These are a bit arcane because they are derived from the data
structures used by Python's 'pgen' parser generator.
There's also a table here mapping operators to their names in the
token module; the Python tokenize module reports all operators as the
fallback token code OP, but the parser needs the actual token code.
i����Ni ( t tokent tokenizet Grammarc B sD e Z d Z d � Z d � Z d � Z d � Z d � Z d � Z RS( s� Pgen parsing tables conversion class.
Once initialized, this class supplies the grammar tables for the
parsing engine implemented by parse.py. The parsing engine
accesses the instance variables directly. The class here does not
provide initialization of the tables; several subclasses exist to
do this (see the conv and pgen modules).
The load() method reads the tables from a pickle file, which is
much faster than the other ways offered by subclasses. The pickle
file is written by calling dump() (after loading the grammar
tables using a subclass). The report() method prints a readable
representation of the tables to stdout, for debugging.
The instance variables are as follows:
symbol2number -- a dict mapping symbol names to numbers. Symbol
numbers are always 256 or higher, to distinguish
them from token numbers, which are between 0 and
255 (inclusive).
number2symbol -- a dict mapping numbers to symbol names;
these two are each other's inverse.
states -- a list of DFAs, where each DFA is a list of
states, each state is a list of arcs, and each
arc is a (i, j) pair where i is a label and j is
a state number. The DFA number is the index into
this list. (This name is slightly confusing.)
Final states are represented by a special arc of
the form (0, j) where j is its own state number.
dfas -- a dict mapping symbol numbers to (DFA, first)
pairs, where DFA is an item from the states list
above, and first is a set of tokens that can
begin this grammar rule (represented by a dict
whose values are always 1).
labels -- a list of (x, y) pairs where x is either a token
number or a symbol number, and y is either None
or a string; the strings are keywords. The label
number is the index in this list; label numbers
are used to mark state transitions (arcs) in the
DFAs.
start -- the number of the grammar's start symbol.
keywords -- a dict mapping keyword strings to arc labels.
tokens -- a dict mapping token numbers to arc labels.
c C sX i | _ i | _ g | _ i | _ d g | _ i | _ i | _ i | _ d | _ d S( Ni t EMPTYi ( i R ( t
symbol2numbert
number2symbolt statest dfast labelst keywordst tokenst symbol2labelt start( t self( ( s- /usr/lib64/python2.7/lib2to3/pgen2/grammar.pyt __init__M s c C s> t | d � �) } t | j � } t j | | d � Wd QXd S( s� Dump the grammar tables to a pickle file.
dump() recursively changes all dict to OrderedDict, so the pickled file
is not exactly the same as what was passed in to dump(). load() uses the
pickled file to create the tables, but only changes OrderedDict to dict
at the top level; it does not recursively change OrderedDict to dict.
So, the loaded tables are different from the original tables that were
passed to load() in that some of the OrderedDict (from the pickled file)
are not changed back to dict. For parsing, this has no effect on
performance because OrderedDict uses dict's __getitem__ with nothing in
between.
t wbi N( t opent _make_deterministict __dict__t picklet dump( R
t filenamet ft d( ( s- /usr/lib64/python2.7/lib2to3/pgen2/grammar.pyR X s
c C s< t | d � } t j | � } | j � | j j | � d S( s+ Load the grammar tables from a pickle file.t rbN( R R t loadt closeR t update( R
R R R ( ( s- /usr/lib64/python2.7/lib2to3/pgen2/grammar.pyR i s
c C s | j j t j | � � d S( s3 Load the grammar tables from a pickle bytes object.N( R R R t loads( R
t pkl( ( s- /usr/lib64/python2.7/lib2to3/pgen2/grammar.pyR p s c C sf | j � } x- d D]% } t | | t | | � j � � q W| j | _ | j | _ | j | _ | S( s#
Copy the grammar.
R R R R R
R ( R R R R R
R ( t __class__t setattrt getattrt copyR R R ( R
t newt dict_attr( ( s- /usr/lib64/python2.7/lib2to3/pgen2/grammar.pyR! t s
#
c C sz d d l m } d GH| | j � d GH| | j � d GH| | j � d GH| | j � d GH| | j � d G| j GHd S(
s: Dump the grammar tables to standard output, for debugging.i����( t pprintt s2nt n2sR R R R N( R$ R R R R R R ( R
R$ ( ( s- /usr/lib64/python2.7/lib2to3/pgen2/grammar.pyt report� s
( t __name__t
__module__t __doc__R R R R R! R' ( ( ( s- /usr/lib64/python2.7/lib2to3/pgen2/grammar.pyR s 4
c C s� t | t � r2 t j t d � | j � D� � � St | t � r^ g | D] } t | � ^ qH St | t � r� t d � | D� � S| S( Nc s s' | ] \ } } | t | � f Vq d S( N( R ( t .0t kt v( ( s- /usr/lib64/python2.7/lib2to3/pgen2/grammar.pys <genexpr>� s c s s | ] } t | � Vq d S( N( R ( R+ t e( ( s- /usr/lib64/python2.7/lib2to3/pgen2/grammar.pys <genexpr>� s ( t
isinstancet dictt collectionst OrderedDictt sortedt iteritemst listR t tuple( t topR. ( ( s- /usr/lib64/python2.7/lib2to3/pgen2/grammar.pyR � s s
( LPAR
) RPAR
[ LSQB
] RSQB
: COLON
, COMMA
; SEMI
+ PLUS
- MINUS
* STAR
/ SLASH
| VBAR
& AMPER
< LESS
> GREATER
= EQUAL
. DOT
% PERCENT
` BACKQUOTE
{ LBRACE
} RBRACE
@ AT
@= ATEQUAL
== EQEQUAL
!= NOTEQUAL
<> NOTEQUAL
<= LESSEQUAL
>= GREATEREQUAL
~ TILDE
^ CIRCUMFLEX
<< LEFTSHIFT
>> RIGHTSHIFT
** DOUBLESTAR
+= PLUSEQUAL
-= MINEQUAL
*= STAREQUAL
/= SLASHEQUAL
%= PERCENTEQUAL
&= AMPEREQUAL
|= VBAREQUAL
^= CIRCUMFLEXEQUAL
<<= LEFTSHIFTEQUAL
>>= RIGHTSHIFTEQUAL
**= DOUBLESTAREQUAL
// DOUBLESLASH
//= DOUBLESLASHEQUAL
-> RARROW
( R* R1 R t R R t objectR R t opmap_rawt opmapt
splitlinest linet splitt opt nameR ( ( ( s- /usr/lib64/python2.7/lib2to3/pgen2/grammar.pyt <module>
s z = __pycache__/pgen.cpython-38.opt-1.pyc 0000644 00000022161 15052147326 0013303 0 ustar 00 U
e5d�5 � @ sd d dl mZmZmZ G dd� dej�ZG dd� de�ZG dd� de�ZG dd � d e�Z dd d �Z
d
S )� )�grammar�token�tokenizec @ s e Zd ZdS )� PgenGrammarN)�__name__�
__module__� __qualname__� r r �*/usr/lib64/python3.8/lib2to3/pgen2/pgen.pyr s r c @ s� e Zd Zd&dd�Zdd� Zdd� Zdd � Zd
d � Zd d
� Zdd� Z dd� Z
dd� Z dd� Z dd� Z
dd� Zdd� Zd d � Zd d � Zd'd d!�Zd"d#� Zd$d%� ZdS )(�ParserGeneratorNc C sl d }|d krt |�}|j}|| _|| _t�|j�| _| �� | � � \| _
| _ |d k rZ|� i | _ | �
� d S �N)�open�close�filename�streamr �generate_tokens�readline� generator�gettoken�parse�dfas� startsymbol�first� addfirstsets)�selfr r Z close_streamr r r
�__init__ s zParserGenerator.__init__c C s t � }t| j�� �}|�� |�| j� |�d| j� |D ]&}dt|j � }||j |< ||j
|< q:|D ]�}| j| }g }|D ]`}g }t |j �
� �D ]$\} }
|�| �|| �|�|
�f� q�|jr�|�d|�|�f� |�|� q||j�|� || �||�f|j|j | < qf|j | j |_|S )N� � )r �listr �keys�sort�remover �insert�len�
symbol2numberZ
number2symbol�sorted�arcs�items�append�
make_label�index�isfinal�states�
make_first�start) r �c�names�name�i�dfar, �stater& �label�nextr r r
� make_grammar s.
z ParserGenerator.make_grammarc C s4 | j | }i }t|�D ]}| �||�}d||< q|S �Nr )r r% r) )r r/ r1 Zrawfirstr r5 �ilabelr r r
r- 4 s
zParserGenerator.make_firstc C s& t |j�}|d �� r�||jkrZ||jkr4|j| S |j�|j| d f� ||j|< |S n>tt|d �}||jkrz|j| S |j�|d f� ||j|< |S n�t |�}|d �� r�||j
kr�|j
| S |j�tj |f� ||j
|< |S n>t j
| }||jk�r|j| S |j�|d f� ||j|< |S d S �Nr )r# �labels�isalphar$ Z symbol2labelr( �getattrr �tokens�eval�keywords�NAMEr Zopmap)r r/ r5 r9 Zitoken�valuer r r
r) = s6
zParserGenerator.make_labelc C s8 t | j�� �}|�� |D ]}|| jkr| �|� qd S r )r r r r r � calcfirst)r r0 r1 r r r
r k s
z ParserGenerator.addfirstsetsc C s� | j | }d | j|< |d }i }i }|j�� D ]x\}}|| j kr�|| jkrj| j| }|d kr~td| ��n| �|� | j| }|�|� |||< q.d||< |di||< q.i } |�� D ]:\}}
|
D ],} | | kr�td|| || | f ��|| | < q�q�|| j|< d S )Nr zrecursion for rule %rr zArule %s is ambiguous; %s is in the first sets of %s as well as %s)r r r& r' �
ValueErrorrC �update) r r1 r3 r4 ZtotalsetZ overlapcheckr5 r6 �fsetZinverseZitsfirstZsymbolr r r
rC s s4
� zParserGenerator.calcfirstc C s� i }d }| j tjkr�| j tjkr*| �� q| �tj�}| �tjd� | �� \}}| �tj� | � ||�}t
|�}| � |� t
|�}|||< |d kr|}q||fS )N�:) �typer � ENDMARKER�NEWLINEr �expectrA �OP� parse_rhs�make_dfar# � simplify_dfa) r r r r1 �a�zr3 ZoldlenZnewlenr r r
r � s"
zParserGenerator.parsec s� � fdd�}� fdd�� t ||�|�g}|D ]�}i }|jD ].}|jD ]"\}} |d k rD� | |�|i �� qDq:t|�� �D ]@\}}
|D ]} | j|
kr� q�q�t |
|�} |�| � |�| |� qvq,|S )Nc s i }� | |� |S r r )r4 �base��
addclosurer r
�closure� s
z)ParserGenerator.make_dfa.<locals>.closurec s: | |kr d S d|| < | j D ]\}}|d kr� ||� qd S r8 �r& )r4 rR r5 r6 rS r r
rT � s z,ParserGenerator.make_dfa.<locals>.addclosure)�DFAState�nfasetr& �
setdefaultr% r' r( �addarc) r r. �finishrU r, r4 r& Znfastater5 r6 rX �str rS r
rN � s"
zParserGenerator.make_dfac
C s� t d|� |g}t|�D ]|\}}t d|||kr2dp4d� |jD ]T\}}||krZ|�|�} nt|�} |�|� |d kr�t d| � q>t d|| f � q>qd S )NzDump of NFA for� State�(final)� z -> %d� %s -> %d)�print� enumerater& r* r# r( )
r r1 r. r[ Ztodor2 r4 r5 r6 �jr r r
�dump_nfa� s
zParserGenerator.dump_nfac C sd t d|� t|�D ]L\}}t d||jr*dp,d� t|j�� �D ] \}}t d||�|�f � q>qd S )NzDump of DFA forr] r^ r_ r` )ra rb r+ r% r& r' r* )r r1 r3 r2 r4 r5 r6 r r r
�dump_dfa� s
zParserGenerator.dump_dfac C sp d}|rld}t |�D ]T\}}t|d t|��D ]8}|| }||kr.||= |D ]}|�||� qLd} qq.qqd S )NTFr )rb �ranger# �
unifystate)r r3 Zchangesr2 Zstate_irc Zstate_jr4 r r r
rO � s z ParserGenerator.simplify_dfac C s~ | � � \}}| jdkr ||fS t� }t� }|�|� |�|� | jdkrr| �� | � � \}}|�|� |�|� q>||fS d S )N�|)� parse_altrB �NFAStaterZ r )r rP rQ ZaaZzzr r r
rM � s
zParserGenerator.parse_rhsc C sL | � � \}}| jdks(| jtjtjfkrD| � � \}}|�|� |}q ||fS )N)�(�[)�
parse_itemrB rH r rA �STRINGrZ )r rP �br/ �dr r r
ri
s
�
zParserGenerator.parse_altc C s� | j dkr>| �� | �� \}}| �tjd� |�|� ||fS | �� \}}| j }|dkr`||fS | �� |�|� |dkr�||fS ||fS d S )Nrl �])�+�*rr )rB r rM rK r rL rZ �
parse_atom)r rP rQ rB r r r
rm s
zParserGenerator.parse_itemc C s� | j dkr4| �� | �� \}}| �tjd� ||fS | jtjtjfkrpt � }t � }|�
|| j � | �� ||fS | � d| j| j � d S )Nrk �)z+expected (...) or NAME or STRING, got %s/%s) rB r rM rK r rL rH rA rn rj rZ � raise_error)r rP rQ r r r
rt ( s
�zParserGenerator.parse_atomc C sD | j |ks |d k r2| j|kr2| �d||| j | j� | j}| �� |S )Nzexpected %s/%s, got %s/%s)rH rB rv r )r rH rB r r r
rK 9 s �zParserGenerator.expectc C sF t | j�}|d tjtjfkr*t | j�}q
|\| _| _| _| _| _ d S r: )
r6 r r �COMMENT�NLrH rB Zbegin�end�line)r �tupr r r
r A s
zParserGenerator.gettokenc
G s^ |r8z || }W n& d� |gttt|�� �}Y nX t|| j| jd | jd | jf��d S )N� r r )�joinr �map�str� SyntaxErrorr ry rz )r �msg�argsr r r
rv H s �zParserGenerator.raise_error)N)N)r r r r r7 r- r) r rC r rN rd re rO rM ri rm rt rK r rv r r r r
r
s$
. $
r c @ s e Zd Zdd� Zddd�ZdS )rj c C s
g | _ d S r rV )r r r r
r S s zNFAState.__init__Nc C s | j �||f� d S r )r& r( �r r6 r5 r r r
rZ V s zNFAState.addarc)N)r r r r rZ r r r r
rj Q s rj c @ s0 e Zd Zdd� Zdd� Zdd� Zdd� Zd Zd S )
rW c C s || _ ||k| _i | _d S r )rX r+ r& )r rX �finalr r r
r ] s
zDFAState.__init__c C s || j |< d S r rV r� r r r
rZ e s zDFAState.addarcc C s* | j �� D ]\}}||kr
|| j |< q
d S r )r& r' )r �old�newr5 r6 r r r
rg k s zDFAState.unifystatec C sV | j |j krdS t| j�t|j�kr(dS | j�� D ] \}}||j�|�k r2 dS q2dS )NFT)r+ r# r& r' �get)r �otherr5 r6 r r r
�__eq__p s zDFAState.__eq__N)r r r r rZ rg r� �__hash__r r r r
rW [ s
rW � Grammar.txtc C s t | �}|�� S r )r r7 )r �pr r r
�generate_grammar� s r� N)r� ) r_ r r r ZGrammarr �objectr rj rW r� r r r r
�<module> s I
% __pycache__/pgen.cpython-38.pyc 0000644 00000023062 15052147326 0012345 0 ustar 00 U
e5d�5 � @ sd d dl mZmZmZ G dd� dej�ZG dd� de�ZG dd� de�ZG dd � d e�Z dd d �Z
d
S )� )�grammar�token�tokenizec @ s e Zd ZdS )� PgenGrammarN)�__name__�
__module__� __qualname__� r r �*/usr/lib64/python3.8/lib2to3/pgen2/pgen.pyr s r c @ s� e Zd Zd&dd�Zdd� Zdd� Zdd � Zd
d � Zd d
� Zdd� Z dd� Z
dd� Z dd� Z dd� Z
dd� Zdd� Zd d � Zd d � Zd'd d!�Zd"d#� Zd$d%� ZdS )(�ParserGeneratorNc C sl d }|d krt |�}|j}|| _|| _t�|j�| _| �� | � � \| _
| _ |d k rZ|� i | _ | �
� d S �N)�open�close�filename�streamr �generate_tokens�readline� generator�gettoken�parse�dfas� startsymbol�first� addfirstsets)�selfr r Z close_streamr r r
�__init__ s zParserGenerator.__init__c C s t � }t| j�� �}|�� |�| j� |�d| j� |D ]&}dt|j � }||j |< ||j
|< q:|D ]�}| j| }g }|D ]`}g }t |j �
� �D ]$\} }
|�| �|| �|�|
�f� q�|jr�|�d|�|�f� |�|� q||j�|� || �||�f|j|j | < qf|j | j |_|S )N� � )r �listr �keys�sort�remover �insert�len�
symbol2numberZ
number2symbol�sorted�arcs�items�append�
make_label�index�isfinal�states�
make_first�start) r �c�names�name�i�dfar, �stater& �label�nextr r r
� make_grammar s.
z ParserGenerator.make_grammarc C s4 | j | }i }t|�D ]}| �||�}d||< q|S �Nr )r r% r) )r r/ r1 Zrawfirstr r5 �ilabelr r r
r- 4 s
zParserGenerator.make_firstc C sb t |j�}|d �� r�||jkrZ||jkr4|j| S |j�|j| d f� ||j|< |S nbtt|d �}t|t �sxt
|��|tj ks�t
|��||j kr�|j | S |j�|d f� ||j |< |S n�|d dks�t
|��t
|�}|d �� �r ||jk� r�|j| S |j�tj|f� ||j|< |S n>tj| }||j k�r@|j | S |j�|d f� ||j |< |S d S )Nr )�"�')r# �labels�isalphar$ Z symbol2labelr( �getattrr �
isinstance�int�AssertionError�tok_name�tokens�eval�keywords�NAMEr Zopmap)r r/ r5 r9 Zitoken�valuer r r
r) = s<
zParserGenerator.make_labelc C s8 t | j�� �}|�� |D ]}|| jkr| �|� qd S r )r r r r r � calcfirst)r r0 r1 r r r
r k s
z ParserGenerator.addfirstsetsc C s� | j | }d | j|< |d }i }i }|j�� D ]x\}}|| j kr�|| jkrj| j| }|d kr~td| ��n| �|� | j| }|�|� |||< q.d||< |di||< q.i } |�� D ]:\}}
|
D ],} | | kr�td|| || | f ��|| | < q�q�|| j|< d S )Nr zrecursion for rule %rr zArule %s is ambiguous; %s is in the first sets of %s as well as %s)r r r& r' �
ValueErrorrH �update) r r1 r3 r4 ZtotalsetZ overlapcheckr5 r6 �fsetZinverseZitsfirstZsymbolr r r
rH s s4
� zParserGenerator.calcfirstc C s� i }d }| j tjkr�| j tjkr*| �� q| �tj�}| �tjd� | �� \}}| �tj� | � ||�}t
|�}| � |� t
|�}|||< |d kr|}q||fS )N�:) �typer � ENDMARKER�NEWLINEr �expectrF �OP� parse_rhs�make_dfar# � simplify_dfa) r r r r1 �a�zr3 ZoldlenZnewlenr r r
r � s"
zParserGenerator.parsec s� t |t�st�t |t�s t�� fdd�}� fdd�� t||�|�g}|D ]�}i }|jD ].}|jD ]"\}} |d k r`� | |�|i �� q`qVt|�� �D ]@\}}
|D ]} | j|
kr� q�q�t|
|�} |� | � |�
| |� q�qH|S )Nc s i }� | |� |S r r )r4 �base��
addclosurer r
�closure� s
z)ParserGenerator.make_dfa.<locals>.closurec sH t | t�st�| |krd S d|| < | jD ]\}}|d kr(� ||� q(d S r8 )r? �NFAStaterA r& )r4 rW r5 r6 rX r r
rY � s z,ParserGenerator.make_dfa.<locals>.addclosure) r? r[ rA �DFAState�nfasetr& �
setdefaultr% r' r( �addarc) r r. �finishrZ r, r4 r& Znfastater5 r6 r] �str rX r
rS � s&
zParserGenerator.make_dfac
C s� t d|� |g}t|�D ]|\}}t d|||kr2dp4d� |jD ]T\}}||krZ|�|�} nt|�} |�|� |d kr�t d| � q>t d|| f � q>qd S )NzDump of NFA for� State�(final)� z -> %d� %s -> %d)�print� enumerater& r* r# r( )
r r1 r. r` Ztodor2 r4 r5 r6 �jr r r
�dump_nfa� s
zParserGenerator.dump_nfac C sd t d|� t|�D ]L\}}t d||jr*dp,d� t|j�� �D ] \}}t d||�|�f � q>qd S )NzDump of DFA forrb rc rd re )rf rg r+ r% r& r' r* )r r1 r3 r2 r4 r5 r6 r r r
�dump_dfa� s
zParserGenerator.dump_dfac C sp d}|rld}t |�D ]T\}}t|d t|��D ]8}|| }||kr.||= |D ]}|�||� qLd} qq.qqd S )NTFr )rg �ranger# �
unifystate)r r3 Zchangesr2 Zstate_irh Zstate_jr4 r r r
rT � s z ParserGenerator.simplify_dfac C s~ | � � \}}| jdkr ||fS t� }t� }|�|� |�|� | jdkrr| �� | � � \}}|�|� |�|� q>||fS d S )N�|)� parse_altrG r[ r_ r )r rU rV ZaaZzzr r r
rR � s
zParserGenerator.parse_rhsc C sL | � � \}}| jdks(| jtjtjfkrD| � � \}}|�|� |}q ||fS )N)�(�[)�
parse_itemrG rM r rF �STRINGr_ )r rU �br/ �dr r r
rn
s
�
zParserGenerator.parse_altc C s� | j dkr>| �� | �� \}}| �tjd� |�|� ||fS | �� \}}| j }|dkr`||fS | �� |�|� |dkr�||fS ||fS d S )Nrp �])�+�*rv )rG r rR rP r rQ r_ �
parse_atom)r rU rV rG r r r
rq s
zParserGenerator.parse_itemc C s� | j dkr4| �� | �� \}}| �tjd� ||fS | jtjtjfkrpt � }t � }|�
|| j � | �� ||fS | � d| j| j � d S )Nro �)z+expected (...) or NAME or STRING, got %s/%s) rG r rR rP r rQ rM rF rr r[ r_ � raise_error)r rU rV r r r
rx ( s
�zParserGenerator.parse_atomc C sD | j |ks |d k r2| j|kr2| �d||| j | j� | j}| �� |S )Nzexpected %s/%s, got %s/%s)rM rG rz r )r rM rG r r r
rP 9 s �zParserGenerator.expectc C sF t | j�}|d tjtjfkr*t | j�}q
|\| _| _| _| _| _ d S )Nr )
r6 r r �COMMENT�NLrM rG Zbegin�end�line)r �tupr r r
r A s
zParserGenerator.gettokenc
G s^ |r8z || }W n& d� |gttt|�� �}Y nX t|| j| jd | jd | jf��d S )N� r r )�joinr �map�str� SyntaxErrorr r} r~ )r �msg�argsr r r
rz H s �zParserGenerator.raise_error)N)N)r r r r r7 r- r) r rH r rS ri rj rT rR rn rq rx rP r rz r r r r
r
s$
. $
r c @ s e Zd Zdd� Zddd�ZdS )r[ c C s
g | _ d S r )r& )r r r r
r S s zNFAState.__init__Nc C s8 |d kst |t�st�t |t�s$t�| j�||f� d S r )r? r� rA r[ r& r( �r r6 r5 r r r
r_ V s zNFAState.addarc)N)r r r r r_ r r r r
r[ Q s r[ c @ s0 e Zd Zdd� Zdd� Zdd� Zdd� Zd Zd S )
r\ c C sL t |t�st�t tt|��t�s$t�t |t�s2t�|| _||k| _i | _d S r ) r? �dictrA r6 �iterr[ r] r+ r& )r r] �finalr r r
r ] s
zDFAState.__init__c C s8 t |t�st�|| jks t�t |t�s*t�|| j|< d S r )r? r� rA r& r\ r� r r r
r_ e s zDFAState.addarcc C s* | j �� D ]\}}||kr
|| j |< q
d S r )r& r' )r �old�newr5 r6 r r r
rl k s zDFAState.unifystatec C sd t |t�st�| j|jkr dS t| j�t|j�kr6dS | j�� D ] \}}||j�|�k r@ dS q@dS )NFT)r? r\ rA r+ r# r& r' �get)r �otherr5 r6 r r r
�__eq__p s zDFAState.__eq__N)r r r r r_ rl r� �__hash__r r r r
r\ [ s
r\ � Grammar.txtc C s t | �}|�� S r )r r7 )r �pr r r
�generate_grammar� s r� N)r� ) rd r r r ZGrammarr �objectr r[ r\ r� r r r r
�<module> s I
% __pycache__/token.cpython-38.opt-1.pyc 0000644 00000003541 15052147326 0013473 0 ustar 00 U
e5d � @ sP d Z dZdZdZdZdZdZdZdZd Z d
Z
d Z d Z d
Z
dZdZdZdZdZdZdZdZdZdZdZdZdZdZd Z d Z d Z d Z d Z d!Z!d"Z"d#Z#d$Z$d%Z%d&Z&d'Z'd(Z(d)Z)d*Z*d+Z+d,Z,d-Z-d.Z.d/Z/d0Z0d1Z1d2Z2d3Z3d4Z4d5Z5d6Z6d7Z7d8Z8d9Z9d:Z:d;Z;d<Z<d=Z=d>Z>i Z?e@eA� �B� �D ]$\ZCZDeEeD�eEd�k�reCe?eD< �qd?d@� ZFdAdB� ZGdCdD� ZHdES )Fz!Token constants (from "token.h").� � � � � � � � � � �
� � �
� � � � � � � � � � � � � � � � � � � �! �" �# �$ �% �& �' �( �) �* �+ �, �- �. �/ �0 �1 �2 �3 �4 �5 �6 �7 �8 �9 �: �; �< � c C s | t k S �N�� NT_OFFSET��x� rD �+/usr/lib64/python3.8/lib2to3/pgen2/token.py�
ISTERMINALO s rF c C s | t kS r? r@ rB rD rD rE �
ISNONTERMINALR s rG c C s | t kS r? )� ENDMARKERrB rD rD rE �ISEOFU s rI N)I�__doc__rH �NAME�NUMBER�STRING�NEWLINE�INDENT�DEDENT�LPAR�RPAR�LSQB�RSQB�COLON�COMMA�SEMI�PLUS�MINUS�STAR�SLASH�VBAR�AMPER�LESS�GREATER�EQUAL�DOT�PERCENTZ BACKQUOTE�LBRACE�RBRACE�EQEQUAL�NOTEQUAL� LESSEQUAL� GREATEREQUAL�TILDE�
CIRCUMFLEX� LEFTSHIFT�
RIGHTSHIFT�
DOUBLESTAR� PLUSEQUAL�MINEQUAL� STAREQUAL�
SLASHEQUAL� PERCENTEQUAL�
AMPEREQUAL� VBAREQUAL�CIRCUMFLEXEQUAL�LEFTSHIFTEQUAL�RIGHTSHIFTEQUAL�DOUBLESTAREQUAL� DOUBLESLASH�DOUBLESLASHEQUAL�AT�ATEQUAL�OP�COMMENT�NL�RARROW�AWAIT�ASYNC�
ERRORTOKEN�
COLONEQUAL�N_TOKENSrA �tok_name�list�globals�items�_nameZ_value�typerF rG rI rD rD rD rE �<module> s� __pycache__/token.cpython-38.opt-2.pyc 0000644 00000003457 15052147326 0013502 0 ustar 00 U
e5d � @ sL d Z dZdZdZdZdZdZdZdZd Z d
Z
d Z d Z d
Z
dZdZdZdZdZdZdZdZdZdZdZdZdZdZd Z d Z d Z d Z d Z d!Z!d"Z"d#Z#d$Z$d%Z%d&Z&d'Z'd(Z(d)Z)d*Z*d+Z+d,Z,d-Z-d.Z.d/Z/d0Z0d1Z1d2Z2d3Z3d4Z4d5Z5d6Z6d7Z7d8Z8d9Z9d:Z:d;Z;d<Z<d=Z=i Z>e?e@� �A� �D ]$\ZBZCeDeC�eDd �k�r
eBe>eC< �q
d>d?� ZEd@dA� ZFdBdC� ZGdDS )E� � � � � � � � � � �
� � �
� � � � � � � � � � � � � � � � � � � �! �" �# �$ �% �& �' �( �) �* �+ �, �- �. �/ �0 �1 �2 �3 �4 �5 �6 �7 �8 �9 �: �; �< � c C s | t k S �N�� NT_OFFSET��x� rD �+/usr/lib64/python3.8/lib2to3/pgen2/token.py�
ISTERMINALO s rF c C s | t kS r? r@ rB rD rD rE �
ISNONTERMINALR s rG c C s | t kS r? )� ENDMARKERrB rD rD rE �ISEOFU s rI N)HrH �NAME�NUMBER�STRING�NEWLINE�INDENT�DEDENT�LPAR�RPAR�LSQB�RSQB�COLON�COMMA�SEMI�PLUS�MINUS�STAR�SLASH�VBAR�AMPER�LESS�GREATER�EQUAL�DOT�PERCENTZ BACKQUOTE�LBRACE�RBRACE�EQEQUAL�NOTEQUAL� LESSEQUAL� GREATEREQUAL�TILDE�
CIRCUMFLEX� LEFTSHIFT�
RIGHTSHIFT�
DOUBLESTAR� PLUSEQUAL�MINEQUAL� STAREQUAL�
SLASHEQUAL� PERCENTEQUAL�
AMPEREQUAL� VBAREQUAL�CIRCUMFLEXEQUAL�LEFTSHIFTEQUAL�RIGHTSHIFTEQUAL�DOUBLESTAREQUAL� DOUBLESLASH�DOUBLESLASHEQUAL�AT�ATEQUAL�OP�COMMENT�NL�RARROW�AWAIT�ASYNC�
ERRORTOKEN�
COLONEQUAL�N_TOKENSrA �tok_name�list�globals�items�_nameZ_value�typerF rG rI rD rD rD rE �<module> s� __pycache__/parse.cpython-38.pyc 0000644 00000014544 15052147326 0012533 0 ustar 00 U
e5d� � @ s4 d Z ddlmZ G dd� de�ZG dd� de�ZdS )z�Parser engine for the grammar tables generated by pgen.
The grammar table must be loaded first.
See Parser/parser.c in the Python distribution for additional info on
how this parsing engine works.
� )�tokenc @ s e Zd ZdZdd� Zdd� ZdS )�
ParseErrorz(Exception to signal the parser is stuck.c C s4 t �| d||||f � || _|| _|| _|| _d S )Nz!%s: type=%r, value=%r, context=%r)� Exception�__init__�msg�type�value�context)�selfr r r r � r �+/usr/lib64/python3.8/lib2to3/pgen2/parse.pyr s
�zParseError.__init__c C s t | �| j| j | j| jffS �N)r r r r )r
r r r �
__reduce__ s zParseError.__reduce__N)�__name__�
__module__� __qualname__�__doc__r r r r r r r s r c @ sL e Zd ZdZddd�Zddd�Zdd� Zd d
� Zd d � Zd
d� Z dd� Z
dS )�Parsera5 Parser engine.
The proper usage sequence is:
p = Parser(grammar, [converter]) # create instance
p.setup([start]) # prepare for parsing
<for each input token>:
if p.addtoken(...): # parse a token; may raise ParseError
break
root = p.rootnode # root of abstract syntax tree
A Parser instance may be reused by calling setup() repeatedly.
A Parser instance contains state pertaining to the current token
sequence, and should not be used concurrently by different threads
to parse separate token sequences.
See driver.py for how to get input tokens by tokenizing a file or
string.
Parsing is complete when addtoken() returns True; the root of the
abstract syntax tree can then be retrieved from the rootnode
instance variable. When a syntax error occurs, addtoken() raises
the ParseError exception. There is no error recovery; the parser
cannot be used after a syntax error was reported (but it can be
reinitialized by calling setup()).
Nc C s || _ |pdd� | _dS )a� Constructor.
The grammar argument is a grammar.Grammar instance; see the
grammar module for more information.
The parser is not ready yet for parsing; you must call the
setup() method to get it started.
The optional convert argument is a function mapping concrete
syntax tree nodes to abstract syntax tree nodes. If not
given, no conversion is done and the syntax tree produced is
the concrete syntax tree. If given, it must be a function of
two arguments, the first being the grammar (a grammar.Grammar
instance), and the second being the concrete syntax tree node
to be converted. The syntax tree is converted from the bottom
up.
A concrete syntax tree node is a (type, value, context, nodes)
tuple, where type is the node type (a token or symbol number),
value is None for symbols and a string for tokens, context is
None or an opaque value used for error reporting (typically a
(lineno, offset) pair), and nodes is a list of children for
symbols, and None for tokens.
An abstract syntax tree node may be anything; this is entirely
up to the converter function.
c S s |S r
r )�grammar�noder r r �<lambda>Z � z!Parser.__init__.<locals>.<lambda>N)r �convert)r
r r r r r r < s zParser.__init__c C sH |dkr| j j}|ddg f}| j j| d|f}|g| _d| _t� | _dS )a� Prepare for parsing.
This *must* be called before starting to parse.
The optional argument is an alternative start symbol; it
defaults to the grammar's start symbol.
You can use a Parser instance to parse any number of programs;
each time you call setup() the parser is reset to an initial
state determined by the (implicit or explicit) start symbol.
N� )r �start�dfas�stack�rootnode�set�
used_names)r
r �newnodeZ
stackentryr r r �setup\ s
z Parser.setupc C s< | � |||�}| jd \}}}|\}} || }
|
D ]�\} } | jj| \}
}|| kr�|
dk s^t�| �||| |� | }|| d|fgkr�| �� | js� dS | jd \}}}|\}} qr dS |
dkr2| jj|
}|\}}||kr2| �|
| jj|
| |� qq2d|f|
k�r(| �� | j�s6t d|||��qt d|||��qdS ) z<Add a token; return True iff this is the end of the program.���� r TFztoo much inputz bad inputN)
�classifyr r �labels�AssertionError�shift�popr �pushr )r
r r r �ilabel�dfa�stater Zstates�firstZarcs�i�newstate�t�vZitsdfaZ itsstatesZitsfirstr r r �addtokent s@
�zParser.addtokenc C sX |t jkr0| j�|� | jj�|�}|dk r0|S | jj�|�}|dkrTtd|||��|S )z&Turn a token into a label. (Internal)Nz bad token) r �NAMEr �addr �keywords�get�tokensr )r
r r r r* r r r r$ � s
zParser.classifyc C sT | j d \}}}|||df}| �| j|�}|dk r@|d �|� |||f| j d< dS )zShift a token. (Internal)r" N)r r r �append) r
r r r/ r r+ r, r r r r r r' � s z Parser.shiftc C sB | j d \}}}|d|g f}|||f| j d< | j �|d|f� dS )z Push a nonterminal. (Internal)r" Nr )r r8 ) r
r Znewdfar/ r r+ r, r r r r r r) � s z Parser.pushc C s` | j �� \}}}| �| j|�}|dk r\| j rL| j d \}}}|d �|� n|| _| j| j_dS )z Pop a nonterminal. (Internal)Nr" )r r( r r r8 r r )r
ZpopdfaZpopstateZpopnoder r+ r, r r r r r( � s z
Parser.pop)N)N) r r r r r r! r2 r$ r' r) r( r r r r r s
0 r N)r � r r r �objectr r r r r �<module> s
__pycache__/tokenize.cpython-38.pyc 0000644 00000035652 15052147326 0013254 0 ustar 00 U
e5d?R �
@ s� d Z dZdZddlZddlZddlmZmZ ddlT ddl m
Z
d d
� e e
�D �d d d
g Z [
ze
W n ek
r~ eZ
Y nX dd� Zdd� Zdd� Zdd� ZdZdZeede � ee� ZdZdZdZd Zed d �Zeeeee�Z d Z ed d!�ee � Z d"e Z ee e �Z ed#e d$ �Z!ee!e e �Z"d%Z#d&Z$d'Z%d(Z&d)Z'ee'd* e'd+ �Z(ee'd, e'd- �Z)ed.d/d0d1d2d3d4d5d6� Z*d7Z+ed8d9d:�Z,ee*e+e,�Z-ee"e-e)e�Z.ee. Z/ee'd; ed<d� e'd= ed>d� �Z0edee(�Z1eee1e"e-e0e� Z2e3ej4e/e2e%e&f�\Z5Z6Z7Z8ed?d@dAdB�ed?d@dCdD�B dEdFdGdHdIdJhB Z9e�4e#�e�4e$�e7e8dK�dLdM� e9D �dNdM� e9D �dOdM� e9D ��Z:d*d+hdPdQ� e9D �B dRdQ� e9D �B Z;d<d>hdSdQ� e9D �B dTdQ� e9D �B Z<dUZ=G dVdW� dWe>�Z?G dXdY� dYe>�Z@dZd[� ZAeAfd\d �ZBd]d^� ZCG d_d`� d`�ZDe�4daejE�ZFe�4dbejE�ZGdcdd� ZHdedf� ZIdgd
� ZJdhd � ZKeLdik�r�ddlMZMeNeMjO�dk�r�eBePeMjOd �jQ� n eBeMjRjQ� dS )ja� Tokenization help for Python programs.
generate_tokens(readline) is a generator that breaks a stream of
text into Python tokens. It accepts a readline-like method which is called
repeatedly to get the next line of input (or "" for EOF). It generates
5-tuples with these members:
the token type (see token.py)
the token (a string)
the starting (row, column) indices of the token (a 2-tuple of ints)
the ending (row, column) indices of the token (a 2-tuple of ints)
the original line (string)
It is designed to match the working of the Python tokenizer exactly, except
that it produces COMMENT tokens for comments and gives type OP for all
operators
Older entry points
tokenize_loop(readline, tokeneater)
tokenize(readline, tokeneater=printtoken)
are the same, except instead of generating tokens, tokeneater is a callback
function to which the 5 fields described above are passed as 5 arguments,
each time a new token is found.zKa-Ping Yee <ping@lfw.org>z@GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, Skip Montanaro� N)�BOM_UTF8�lookup)�*� )�tokenc C s g | ]}|d dkr|�qS )r �_� )�.0�xr r �./usr/lib64/python3.8/lib2to3/pgen2/tokenize.py�
<listcomp>% s r �tokenize�generate_tokens�
untokenizec G s dd� | � d S )N�(�|�))�join��choicesr r r �group0 � r c G s t | � d S )Nr �r r r r r �any1 r r c G s t | � d S )N�?r r r r r �maybe2 r r c s t � fdd�� D ��S )Nc 3 s4 | ],}� d D ] }|� � |� � kr|| V qqdS ))� N)�casefold)r r
�y��lr r � <genexpr>4 s
z _combinations.<locals>.<genexpr>)�setr r r r �
_combinations3 s �r# z[ \f\t]*z #[^\r\n]*z\\\r?\nz\w+z0[bB]_?[01]+(?:_[01]+)*z(0[xX]_?[\da-fA-F]+(?:_[\da-fA-F]+)*[lL]?z 0[oO]?_?[0-7]+(?:_[0-7]+)*[lL]?z[1-9]\d*(?:_\d+)*[lL]?z0[lL]?z[eE][-+]?\d+(?:_\d+)*z \d+(?:_\d+)*\.(?:\d+(?:_\d+)*)?z\.\d+(?:_\d+)*z \d+(?:_\d+)*z\d+(?:_\d+)*[jJ]z[jJ]z[^'\\]*(?:\\.[^'\\]*)*'z[^"\\]*(?:\\.[^"\\]*)*"z%[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''z%[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""z'(?:[uUrRbBfF]|[rR][fFbB]|[fFbBuU][rR])?�'''�"""z '[^\n'\\]*(?:\\.[^\n'\\]*)*'z "[^\n"\\]*(?:\\.[^\n"\\]*)*"z\*\*=?z>>=?z<<=?z<>z!=z//=?z->z[+\-*/%&@|^=<>]=?�~z[][(){}]z\r?\nz:=z[:;.,`@]z'[^\n'\\]*(?:\\.[^\n'\\]*)*�'z"[^\n"\\]*(?:\\.[^\n"\\]*)*�"�r�R�f�F�b�B�u�UZurZuRZUrZUR)r' r( r$ r% c C s i | ]}|� d �t �qS �r$ )� single3prog�r �prefixr r r �
<dictcomp>y s r5 c C s i | ]}|� d �t �qS �r% )� double3progr3 r r r r5 z s c C s i | ]
}|d �qS �Nr r3 r r r r5 { s c C s h | ]}|� d ��qS r1 r r3 r r r � <setcomp> s r9 c C s h | ]}|� d ��qS r6 r r3 r r r r9 � s c C s h | ]}|� d ��qS )r' r r3 r r r r9 � s c C s h | ]}|� d ��qS )r( r r3 r r r r9 � s � c @ s e Zd ZdS )�
TokenErrorN��__name__�
__module__� __qualname__r r r r r; � s r; c @ s e Zd ZdS )�StopTokenizingNr< r r r r r@ � s r@ c C s4 |\}}|\}}t d||||t| t|�f � d S )Nz%d,%d-%d,%d: %s %s)�print�tok_name�repr) �typer Zxxx_todo_changemeZxxx_todo_changeme1�lineZsrowZscolZerowZecolr r r �
printtoken� s
�rF c C s( zt | |� W n tk
r" Y nX dS )a:
The tokenize() function accepts two parameters: one representing the
input stream, and one providing an output mechanism for tokenize().
The first parameter, readline, must be a callable object which provides
the same interface as the readline() method of built-in file objects.
Each call to the function should return one line of input as a string.
The second parameter, tokeneater, must also be a callable object. It is
called once for each token, with five arguments, corresponding to the
tuples generated by generate_tokens().
N)�
tokenize_loopr@ )�readline�
tokeneaterr r r r
� s
c C s t | �D ] }||� qd S r8 )r )rH rI Z
token_infor r r rG � s rG c @ s, e Zd Zdd� Zdd� Zdd� Zdd� Zd S )
� Untokenizerc C s g | _ d| _d| _d S )Nr r )�tokens�prev_row�prev_col)�selfr r r �__init__� s zUntokenizer.__init__c C s8 |\}}|| j kst�|| j }|r4| j�d| � d S )N� )rL �AssertionErrorrM rK �append)rN �start�row�col�
col_offsetr r r �add_whitespace� s
zUntokenizer.add_whitespacec C s� |D ]p}t |�dkr$| �||� qv|\}}}}}| �|� | j�|� |\| _| _|ttfkr| jd7 _d| _qd� | j�S )N� r r r )
�len�compatrW rK rR rL rM �NEWLINE�NLr )rN �iterable�t�tok_typer rS �endrE r r r r � s
zUntokenizer.untokenizec C s� d}g }| j j}|\}}|ttfkr,|d7 }|ttfkr<d}|D ]�}|d d� \}}|ttttfkrl|d7 }|tkr�|�|� q@n>|t kr�|�
� q@n*|ttfkr�d}n|r�|r�||d � d}||� q@d S )NFrP TrX ���) rK rR �NAME�NUMBERr[ r\ �ASYNC�AWAIT�INDENT�DEDENT�pop) rN r r] � startline�indents� toks_append�toknum�tokval�tokr r r rZ � s0
zUntokenizer.compatN)r= r> r? rO rW r rZ r r r r rJ � s rJ z&^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)s ^[ \t\f]*(?:[#\r\n]|$)c C sH | dd� � � �dd�}|dks*|�d�r.dS |dks@|�d�rDd S | S )
z(Imitates get_normal_name in tokenizer.c.N� r �-�utf-8zutf-8-)zlatin-1�
iso-8859-1z iso-latin-1)zlatin-1-z iso-8859-1-z iso-latin-1-rr )�lower�replace�
startswith)�orig_enc�encr r r �_get_normal_name� s �rx c s� d� d}d}�fdd�}� fdd�}|� }|� t�rHd� |d d� }d
}|sT|g fS ||�}|rj||gfS t�|�s~||gfS |� }|s�||gfS ||�}|r�|||gfS |||gfS ) a
The detect_encoding() function is used to detect the encoding that should
be used to decode a Python source file. It requires one argument, readline,
in the same way as the tokenize() generator.
It will call readline a maximum of twice, and return the encoding used
(as a string) and a list of any lines (left as bytes) it has read
in.
It detects the encoding from the presence of a utf-8 bom or an encoding
cookie as specified in pep-0263. If both a bom and a cookie are present, but
disagree, a SyntaxError will be raised. If the encoding cookie is an invalid
charset, raise a SyntaxError. Note that if a utf-8 bom is found,
'utf-8-sig' is returned.
If no encoding is specified, then the default of 'utf-8' will be returned.
FNrq c s( z� � W S t k
r" t� Y S X d S r8 )�
StopIteration�bytesr )rH r r � read_or_stop s z%detect_encoding.<locals>.read_or_stopc s� z| � d�}W n tk
r$ Y d S X t�|�}|s8d S t|�d��}z t|�}W n tk
rr td| ��Y nX � r�|j dkr�td��|d7 }|S )N�asciir zunknown encoding: rq zencoding problem: utf-8z-sig)
�decode�UnicodeDecodeError� cookie_re�matchrx r r � LookupError� SyntaxError�name)rE � line_stringr� �encoding�codec)� bom_foundr r � find_cookie s"
z$detect_encoding.<locals>.find_cookieT� z utf-8-sig)ru r �blank_rer� )rH r� �defaultr{ r� �first�secondr )r� rH r �detect_encoding� s0
r� c C s t � }|�| �S )a� Transform tokens back into Python source code.
Each element returned by the iterable must be a token sequence
with at least two elements, a token number and token value. If
only two tokens are passed, the resulting output is poor.
Round-trip invariant for full input:
Untokenized source will match input source exactly
Round-trip invariant for limited input:
# Output text will tokenize the back to the input
t1 = [tok[:2] for tok in generate_tokens(f.readline)]
newcode = untokenize(t1)
readline = iter(newcode.splitlines(1)).next
t2 = [tok[:2] for tokin generate_tokens(readline)]
assert t1 == t2
)rJ r )r] �utr r r r : s c c s� d } }}d\}}d}dg}d}d} d}
d} z
| � } W n t k
rP d} Y nX |d }dt| � }
}|�r2| s|td|��|�| �}|r�|�d� }
}t|| d|� |||f|| fV d\}}d}nd|�r | dd� d k�r | d
d� d k�r t|| ||t| �f|fV d}d}q.n|| }|| }q.�nB|dk�r\|�s\| �sL�q,d}|
|k �r�| |
d k�rr|d }n8| |
d
k�r�|t d t }n| |
dk�r�d}n�q�|
d }
�qP|
|k�rĐq,|�r�|V d}| |
dk�r�| |
dk�rT| |
d� �d�}|
t|� }t |||
f||
t|� f| fV t
| |d� ||f|t| �f| fV q.t
t f| |
dk | |
d� ||
f|t| �f| fV q.||d k�r�|� |� t | d|
� |df||
f| fV ||d k �r4||k�r�t
dd||
| f��|dd� }| �r|
|d k�rd} d} d}
td||
f||
f| fV �q�| �rt| �rt|
|d k�rtd} d} d}
n| �sptd|df��d}|
|k r.t�| |
�}|�r�|�d�\}}||f||f| }}}
| ||� | | }}|tjk�s�|dk�r�|dk�r�t|||| fV �q&|dk�rJt} |dk�r t
} n
| �r&d} |�r6|V d}| |||| fV �q&|dk�r�|�d��rdt�|�rt|V d}t |||| fV �q&|tk�rt| }|�| |
�}|�r�|�d�}
| ||
� }|�r�|V d}t||||
f| fV n||f}| |d� }| }q.�q&|tk�s4|dd� tk�s4|dd� tk�r�|d dk�r�||f}t| �plt|d �plt|d }| |d� d }}| }q.n |�r�|V d}t|||| fV �q&|�� �r�|dk�r�| �r�|d k�r�tnt |||| fV �qtt |||| f} |d k�r|�s| }�qt|d k�rj|�rj|d t k�rj|d d k�rjd} |d }
t|d |d |d |d fV d}|�rz|V d}| V nz|d k�r�|�r�|V d}t
||||
f| fV d}nF|d k�r�|d }n|d!k�r�|d }|�r�|V d}t |||| fV n(t| |
||
f||
d f| fV |
d }
�qtq.|�r<|V d}|dd� D ] } td|df|dfdfV �qHt d|df|dfdfV dS )"a4
The generate_tokens() generator requires one argument, readline, which
must be a callable object which provides the same interface as the
readline() method of built-in file objects. Each call to the function
should return one line of input as a string. Alternately, readline
can be a callable function terminating with StopIteration:
readline = open(myfile).next # Example of alternate readline
The generator produces 5-tuples with these members: the token type; the
token string; a 2-tuple (srow, scol) of ints specifying the row and
column where the token begins in the source; a 2-tuple (erow, ecol) of
ints specifying the row and column where the token ends in the source;
and the line on which the token was found. The line passed is the
physical line.
r )r r NFr r zEOF in multi-line string���z\
���z\
rP � � z#
�#z
ra z3unindent does not match any outer indentation levelz
<tokenize>zEOF in multi-line statement�.T�
rX r� )�async�awaitr� �def� �\z([{z)]}) ry rY r; r� r` �STRING�
ERRORTOKEN�tabsize�rstrip�COMMENTr\ rR rf �IndentationErrorrg �
pseudoprog�span�stringZdigitsrc r[ �endswithrQ �
triple_quoted�endprogs�
single_quoted� isidentifierrd re rb �OP� ENDMARKER) rH �lnum�parenlev� continued�contstr�needcont�contlinerj ZstashedZ async_defZasync_def_indentZ async_def_nlrE �pos�max�strstart�endprog�endmatchr` �column�
comment_tokenZnl_pos� pseudomatchrS �spos�eposr �initial�newlinern �indentr r r r O s�
�*
�
�
�
�
� ��
��
�
�
�
� �
� �__main__)S�__doc__�
__author__� __credits__r� �re�codecsr r Zlib2to3.pgen2.tokenr r �dir�__all__rz � NameError�strr r r r# �
Whitespace�Comment�Ignore�Name� Binnumber� Hexnumber� Octnumber� Decnumber� Intnumber�Exponent�
Pointfloat�Expfloat� Floatnumber�
Imagnumber�Number�Single�Double�Single3�Double3Z
_litprefix�Triple�StringZOperatorZBracket�Special�Funny�
PlainToken�Token�ContStr� PseudoExtras� PseudoToken�map�compileZ tokenprogr� r2 r7 Z _strprefixesr� r� r� r� � Exceptionr; r@ rF r
rG rJ �ASCIIr r� rx r� r r r= �sysrY �argv�openrH �stdinr r r r �<module> s� � �
� � ���
� ��� � � � �� � �� 8 I b
__pycache__/tokenize.cpython-38.opt-1.pyc 0000644 00000035544 15052147326 0014213 0 ustar 00 U
e5d?R �
@ s� d Z dZdZddlZddlZddlmZmZ ddlT ddl m
Z
d d
� e e
�D �d d d
g Z [
ze
W n ek
r~ eZ
Y nX dd� Zdd� Zdd� Zdd� ZdZdZeede � ee� ZdZdZdZd Zed d �Zeeeee�Z d Z ed d!�ee � Z d"e Z ee e �Z ed#e d$ �Z!ee!e e �Z"d%Z#d&Z$d'Z%d(Z&d)Z'ee'd* e'd+ �Z(ee'd, e'd- �Z)ed.d/d0d1d2d3d4d5d6� Z*d7Z+ed8d9d:�Z,ee*e+e,�Z-ee"e-e)e�Z.ee. Z/ee'd; ed<d� e'd= ed>d� �Z0edee(�Z1eee1e"e-e0e� Z2e3ej4e/e2e%e&f�\Z5Z6Z7Z8ed?d@dAdB�ed?d@dCdD�B dEdFdGdHdIdJhB Z9e�4e#�e�4e$�e7e8dK�dLdM� e9D �dNdM� e9D �dOdM� e9D ��Z:d*d+hdPdQ� e9D �B dRdQ� e9D �B Z;d<d>hdSdQ� e9D �B dTdQ� e9D �B Z<dUZ=G dVdW� dWe>�Z?G dXdY� dYe>�Z@dZd[� ZAeAfd\d �ZBd]d^� ZCG d_d`� d`�ZDe�4daejE�ZFe�4dbejE�ZGdcdd� ZHdedf� ZIdgd
� ZJdhd � ZKeLdik�r�ddlMZMeNeMjO�dk�r�eBePeMjOd �jQ� n eBeMjRjQ� dS )ja� Tokenization help for Python programs.
generate_tokens(readline) is a generator that breaks a stream of
text into Python tokens. It accepts a readline-like method which is called
repeatedly to get the next line of input (or "" for EOF). It generates
5-tuples with these members:
the token type (see token.py)
the token (a string)
the starting (row, column) indices of the token (a 2-tuple of ints)
the ending (row, column) indices of the token (a 2-tuple of ints)
the original line (string)
It is designed to match the working of the Python tokenizer exactly, except
that it produces COMMENT tokens for comments and gives type OP for all
operators
Older entry points
tokenize_loop(readline, tokeneater)
tokenize(readline, tokeneater=printtoken)
are the same, except instead of generating tokens, tokeneater is a callback
function to which the 5 fields described above are passed as 5 arguments,
each time a new token is found.zKa-Ping Yee <ping@lfw.org>z@GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, Skip Montanaro� N)�BOM_UTF8�lookup)�*� )�tokenc C s g | ]}|d dkr|�qS )r �_� )�.0�xr r �./usr/lib64/python3.8/lib2to3/pgen2/tokenize.py�
<listcomp>% s r �tokenize�generate_tokens�
untokenizec G s dd� | � d S )N�(�|�))�join��choicesr r r �group0 � r c G s t | � d S )Nr �r r r r r �any1 r r c G s t | � d S )N�?r r r r r �maybe2 r r c s t � fdd�� D ��S )Nc 3 s4 | ],}� d D ] }|� � |� � kr|| V qqdS ))� N)�casefold)r r
�y��lr r � <genexpr>4 s
z _combinations.<locals>.<genexpr>)�setr r r r �
_combinations3 s �r# z[ \f\t]*z #[^\r\n]*z\\\r?\nz\w+z0[bB]_?[01]+(?:_[01]+)*z(0[xX]_?[\da-fA-F]+(?:_[\da-fA-F]+)*[lL]?z 0[oO]?_?[0-7]+(?:_[0-7]+)*[lL]?z[1-9]\d*(?:_\d+)*[lL]?z0[lL]?z[eE][-+]?\d+(?:_\d+)*z \d+(?:_\d+)*\.(?:\d+(?:_\d+)*)?z\.\d+(?:_\d+)*z \d+(?:_\d+)*z\d+(?:_\d+)*[jJ]z[jJ]z[^'\\]*(?:\\.[^'\\]*)*'z[^"\\]*(?:\\.[^"\\]*)*"z%[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''z%[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""z'(?:[uUrRbBfF]|[rR][fFbB]|[fFbBuU][rR])?�'''�"""z '[^\n'\\]*(?:\\.[^\n'\\]*)*'z "[^\n"\\]*(?:\\.[^\n"\\]*)*"z\*\*=?z>>=?z<<=?z<>z!=z//=?z->z[+\-*/%&@|^=<>]=?�~z[][(){}]z\r?\nz:=z[:;.,`@]z'[^\n'\\]*(?:\\.[^\n'\\]*)*�'z"[^\n"\\]*(?:\\.[^\n"\\]*)*�"�r�R�f�F�b�B�u�UZurZuRZUrZUR)r' r( r$ r% c C s i | ]}|� d �t �qS �r$ )� single3prog�r �prefixr r r �
<dictcomp>y s r5 c C s i | ]}|� d �t �qS �r% )� double3progr3 r r r r5 z s c C s i | ]
}|d �qS �Nr r3 r r r r5 { s c C s h | ]}|� d ��qS r1 r r3 r r r � <setcomp> s r9 c C s h | ]}|� d ��qS r6 r r3 r r r r9 � s c C s h | ]}|� d ��qS )r' r r3 r r r r9 � s c C s h | ]}|� d ��qS )r( r r3 r r r r9 � s � c @ s e Zd ZdS )�
TokenErrorN��__name__�
__module__� __qualname__r r r r r; � s r; c @ s e Zd ZdS )�StopTokenizingNr< r r r r r@ � s r@ c C s4 |\}}|\}}t d||||t| t|�f � d S )Nz%d,%d-%d,%d: %s %s)�print�tok_name�repr) �typer Zxxx_todo_changemeZxxx_todo_changeme1�lineZsrowZscolZerowZecolr r r �
printtoken� s
�rF c C s( zt | |� W n tk
r" Y nX dS )a:
The tokenize() function accepts two parameters: one representing the
input stream, and one providing an output mechanism for tokenize().
The first parameter, readline, must be a callable object which provides
the same interface as the readline() method of built-in file objects.
Each call to the function should return one line of input as a string.
The second parameter, tokeneater, must also be a callable object. It is
called once for each token, with five arguments, corresponding to the
tuples generated by generate_tokens().
N)�
tokenize_loopr@ )�readline�
tokeneaterr r r r
� s
c C s t | �D ] }||� qd S r8 )r )rH rI Z
token_infor r r rG � s rG c @ s, e Zd Zdd� Zdd� Zdd� Zdd� Zd S )
� Untokenizerc C s g | _ d| _d| _d S )Nr r )�tokens�prev_row�prev_col)�selfr r r �__init__� s zUntokenizer.__init__c C s* |\}}|| j }|r&| j�d| � d S )N� )rM rK �append)rN �start�row�col�
col_offsetr r r �add_whitespace� s
zUntokenizer.add_whitespacec C s� |D ]p}t |�dkr$| �||� qv|\}}}}}| �|� | j�|� |\| _| _|ttfkr| jd7 _d| _qd� | j�S )N� r r r )
�len�compatrV rK rQ rL rM �NEWLINE�NLr )rN �iterable�t�tok_typer rR �endrE r r r r � s
zUntokenizer.untokenizec C s� d}g }| j j}|\}}|ttfkr,|d7 }|ttfkr<d}|D ]�}|d d� \}}|ttttfkrl|d7 }|tkr�|�|� q@n>|t kr�|�
� q@n*|ttfkr�d}n|r�|r�||d � d}||� q@d S )NFrP TrW ���) rK rQ �NAME�NUMBERrZ r[ �ASYNC�AWAIT�INDENT�DEDENT�pop) rN r r\ � startline�indents� toks_append�toknum�tokval�tokr r r rY � s0
zUntokenizer.compatN)r= r> r? rO rV r rY r r r r rJ � s rJ z&^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)s ^[ \t\f]*(?:[#\r\n]|$)c C sH | dd� � � �dd�}|dks*|�d�r.dS |dks@|�d�rDd S | S )
z(Imitates get_normal_name in tokenizer.c.N� r �-�utf-8zutf-8-)zlatin-1�
iso-8859-1z iso-latin-1)zlatin-1-z iso-8859-1-z iso-latin-1-rq )�lower�replace�
startswith)�orig_enc�encr r r �_get_normal_name� s �rw c s� d� d}d}�fdd�}� fdd�}|� }|� t�rHd� |d d� }d
}|sT|g fS ||�}|rj||gfS t�|�s~||gfS |� }|s�||gfS ||�}|r�|||gfS |||gfS ) a
The detect_encoding() function is used to detect the encoding that should
be used to decode a Python source file. It requires one argument, readline,
in the same way as the tokenize() generator.
It will call readline a maximum of twice, and return the encoding used
(as a string) and a list of any lines (left as bytes) it has read
in.
It detects the encoding from the presence of a utf-8 bom or an encoding
cookie as specified in pep-0263. If both a bom and a cookie are present, but
disagree, a SyntaxError will be raised. If the encoding cookie is an invalid
charset, raise a SyntaxError. Note that if a utf-8 bom is found,
'utf-8-sig' is returned.
If no encoding is specified, then the default of 'utf-8' will be returned.
FNrp c s( z� � W S t k
r" t� Y S X d S r8 )�
StopIteration�bytesr )rH r r � read_or_stop s z%detect_encoding.<locals>.read_or_stopc s� z| � d�}W n tk
r$ Y d S X t�|�}|s8d S t|�d��}z t|�}W n tk
rr td| ��Y nX � r�|j dkr�td��|d7 }|S )N�asciir zunknown encoding: rp zencoding problem: utf-8z-sig)
�decode�UnicodeDecodeError� cookie_re�matchrw r r � LookupError� SyntaxError�name)rE � line_stringr �encoding�codec)� bom_foundr r � find_cookie s"
z$detect_encoding.<locals>.find_cookieT� z utf-8-sig)rt r �blank_rer )rH r� �defaultrz r� �first�secondr )r� rH r �detect_encoding� s0
r� c C s t � }|�| �S )a� Transform tokens back into Python source code.
Each element returned by the iterable must be a token sequence
with at least two elements, a token number and token value. If
only two tokens are passed, the resulting output is poor.
Round-trip invariant for full input:
Untokenized source will match input source exactly
Round-trip invariant for limited input:
# Output text will tokenize the back to the input
t1 = [tok[:2] for tok in generate_tokens(f.readline)]
newcode = untokenize(t1)
readline = iter(newcode.splitlines(1)).next
t2 = [tok[:2] for tokin generate_tokens(readline)]
assert t1 == t2
)rJ r )r\ �utr r r r : s c c st d } }}d\}}d}dg}d}d} d}
d} z
| � } W n t k
rP d} Y nX |d }dt| � }
}|�r2| s|td|��|�| �}|r�|�d� }
}t|| d|� |||f|| fV d\}}d}nd|�r | dd� d k�r | d
d� d k�r t|| ||t| �f|fV d}d}q.n|| }|| }q.�nB|dk�r\|�s\| �sL�q d}|
|k �r�| |
d k�rr|d }n8| |
d
k�r�|t d t }n| |
dk�r�d}n�q�|
d }
�qP|
|k�rĐq |�r�|V d}| |
dk�r�| |
dk�rT| |
d� �d�}|
t|� }t |||
f||
t|� f| fV t
| |d� ||f|t| �f| fV q.t
t f| |
dk | |
d� ||
f|t| �f| fV q.||d k�r�|� |� t | d|
� |df||
f| fV ||d k �r4||k�r�t
dd||
| f��|dd� }| �r|
|d k�rd} d} d}
td||
f||
f| fV �q�| �rt| �rt|
|d k�rtd} d} d}
n| �sptd|df��d}|
|k r.t�| |
�}|�r�|�d�\}}||f||f| }}}
| ||� | | }}|tjk�s�|dk�r�|dk�r�t|||| fV �q|dk�rJt} |dk�r t
} n
| �r&d} |�r6|V d}| |||| fV �q|dk�rx|�rd|V d}t |||| fV �q|tk�r�t| }|�| |
�}|�r�|�d�}
| ||
� }|�r�|V d}t||||
f| fV n||f}| |d� }| }q.�q|tk�s$|dd� tk�s$|dd� tk�r�|d dk�rx||f}t| �p\t|d �p\t|d }| |d� d }}| }q.n |�r�|V d}t|||| fV �q|�� �rr|dk�r�| �r�|d k�r�tnt|||| fV �qtt|||| f} |d k�r�|�s�| }�qt|d k�rZ|�rZ|d tk�rZ|d d k�rZd} |d }
t|d |d |d |d fV d}|�rj|V d}| V nz|d k�r�|�r�|V d}t
||||
f| fV d}nF|d k�r�|d }n|d!k�r�|d }|�r�|V d}t |||| fV n(t| |
||
f||
d f| fV |
d }
�qtq.|�r,|V d}|dd� D ] } td|df|dfdfV �q8t d|df|dfdfV dS )"a4
The generate_tokens() generator requires one argument, readline, which
must be a callable object which provides the same interface as the
readline() method of built-in file objects. Each call to the function
should return one line of input as a string. Alternately, readline
can be a callable function terminating with StopIteration:
readline = open(myfile).next # Example of alternate readline
The generator produces 5-tuples with these members: the token type; the
token string; a 2-tuple (srow, scol) of ints specifying the row and
column where the token begins in the source; a 2-tuple (erow, ecol) of
ints specifying the row and column where the token ends in the source;
and the line on which the token was found. The line passed is the
physical line.
r )r r NFr r zEOF in multi-line string���z\
���z\
rP � � z#
�#z
r` z3unindent does not match any outer indentation levelz
<tokenize>zEOF in multi-line statement�.TrW r� �
)�async�awaitr� �def� �\z([{z)]}) rx rX r; r r_ �STRING�
ERRORTOKEN�tabsize�rstrip�COMMENTr[ rQ re �IndentationErrorrf �
pseudoprog�span�stringZdigitsrb rZ �
triple_quoted�endprogs�
single_quoted� isidentifierrc rd ra �OP� ENDMARKER) rH �lnum�parenlev� continued�contstr�needcont�contlineri ZstashedZ async_defZasync_def_indentZ async_def_nlrE �pos�max�strstart�endprog�endmatchr_ �column�
comment_tokenZnl_pos� pseudomatchrR �spos�eposr �initial�newlinerm �indentr r r r O s�
�*
�
�
�
�
� ��
��
�
�
�
� �
� �__main__)S�__doc__�
__author__� __credits__r� �re�codecsr r Zlib2to3.pgen2.tokenr r �dir�__all__ry � NameError�strr r r r# �
Whitespace�Comment�Ignore�Name� Binnumber� Hexnumber� Octnumber� Decnumber� Intnumber�Exponent�
Pointfloat�Expfloat� Floatnumber�
Imagnumber�Number�Single�Double�Single3�Double3Z
_litprefix�Triple�StringZOperatorZBracket�Special�Funny�
PlainToken�Token�ContStr� PseudoExtras� PseudoToken�map�compileZ tokenprogr� r2 r7 Z _strprefixesr� r� r� r� � Exceptionr; r@ rF r
rG rJ �ASCIIr~ r� rw r� r r r= �sysrX �argv�openrH �stdinr r r r �<module> s� � �
� � ���
� ��� � � � �� � �� 8 I b
__pycache__/__init__.cpython-38.opt-1.pyc 0000644 00000000247 15052147326 0014112 0 ustar 00 U
e5d� � @ s d Z dS )zThe pgen2 package.N)�__doc__� r r �./usr/lib64/python3.8/lib2to3/pgen2/__init__.py�<module> � __pycache__/tokenize.cpython-38.opt-2.pyc 0000644 00000026056 15052147326 0014212 0 ustar 00 U
e5d?R �
@ s� d Z dZddlZddlZddlmZmZ ddlT ddlm Z dd � e
e �D �d
d d g Z [ ze W n e
k
rz eZ Y nX d
d� Zdd� Zdd� Zdd� ZdZdZeede � ee� ZdZdZdZdZed d �Zeeeee�Zd Z ed d �ee � Z d!e Z ee e �Z ed"e d# �Z ee e e�Z!d$Z"d%Z#d&Z$d'Z%d(Z&ee&d) e&d* �Z'ee&d+ e&d, �Z(ed-d.d/d0d1d2d3d4d5� Z)d6Z*ed7d8d9�Z+ee)e*e+�Z,ee!e,e(e�Z-ee- Z.ee&d: ed;d� e&d< ed=d� �Z/edee'�Z0eee0e!e,e/e� Z1e2ej3e.e1e$e%f�\Z4Z5Z6Z7ed>d?d@dA�ed>d?dBdC�B dDdEdFdGdHdIhB Z8e�3e"�e�3e#�e6e7dJ�dKdL� e8D �dMdL� e8D �dNdL� e8D ��Z9d)d*hdOdP� e8D �B dQdP� e8D �B Z:d;d=hdRdP� e8D �B dSdP� e8D �B Z;dTZ<G dUdV� dVe=�Z>G dWdX� dXe=�Z?dYdZ� Z@e@fd[d
�ZAd\d]� ZBG d^d_� d_�ZCe�3d`ejD�ZEe�3daejD�ZFdbdc� ZGddde� ZHdfd � ZIdgd � ZJeKdhk�r�ddlLZLeMeLjN�dk�r�eAeOeLjNd �jP� n eAeLjQjP� dS )izKa-Ping Yee <ping@lfw.org>z@GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, Skip Montanaro� N)�BOM_UTF8�lookup)�*� )�tokenc C s g | ]}|d dkr|�qS )r �_� )�.0�xr r �./usr/lib64/python3.8/lib2to3/pgen2/tokenize.py�
<listcomp>% s r �tokenize�generate_tokens�
untokenizec G s dd� | � d S )N�(�|�))�join��choicesr r r �group0 � r c G s t | � d S )Nr �r r r r r �any1 r r c G s t | � d S )N�?r r r r r �maybe2 r r c s t � fdd�� D ��S )Nc 3 s4 | ],}� d D ] }|� � |� � kr|| V qqdS ))� N)�casefold)r r
�y��lr r � <genexpr>4 s
z _combinations.<locals>.<genexpr>)�setr r r r �
_combinations3 s �r# z[ \f\t]*z #[^\r\n]*z\\\r?\nz\w+z0[bB]_?[01]+(?:_[01]+)*z(0[xX]_?[\da-fA-F]+(?:_[\da-fA-F]+)*[lL]?z 0[oO]?_?[0-7]+(?:_[0-7]+)*[lL]?z[1-9]\d*(?:_\d+)*[lL]?z0[lL]?z[eE][-+]?\d+(?:_\d+)*z \d+(?:_\d+)*\.(?:\d+(?:_\d+)*)?z\.\d+(?:_\d+)*z \d+(?:_\d+)*z\d+(?:_\d+)*[jJ]z[jJ]z[^'\\]*(?:\\.[^'\\]*)*'z[^"\\]*(?:\\.[^"\\]*)*"z%[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''z%[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""z'(?:[uUrRbBfF]|[rR][fFbB]|[fFbBuU][rR])?�'''�"""z '[^\n'\\]*(?:\\.[^\n'\\]*)*'z "[^\n"\\]*(?:\\.[^\n"\\]*)*"z\*\*=?z>>=?z<<=?z<>z!=z//=?z->z[+\-*/%&@|^=<>]=?�~z[][(){}]z\r?\nz:=z[:;.,`@]z'[^\n'\\]*(?:\\.[^\n'\\]*)*�'z"[^\n"\\]*(?:\\.[^\n"\\]*)*�"�r�R�f�F�b�B�u�UZurZuRZUrZUR)r' r( r$ r% c C s i | ]}|� d �t �qS �r$ )� single3prog�r �prefixr r r �
<dictcomp>y s r5 c C s i | ]}|� d �t �qS �r% )� double3progr3 r r r r5 z s c C s i | ]
}|d �qS �Nr r3 r r r r5 { s c C s h | ]}|� d ��qS r1 r r3 r r r � <setcomp> s r9 c C s h | ]}|� d ��qS r6 r r3 r r r r9 � s c C s h | ]}|� d ��qS )r' r r3 r r r r9 � s c C s h | ]}|� d ��qS )r( r r3 r r r r9 � s � c @ s e Zd ZdS )�
TokenErrorN��__name__�
__module__� __qualname__r r r r r; � s r; c @ s e Zd ZdS )�StopTokenizingNr< r r r r r@ � s r@ c C s4 |\}}|\}}t d||||t| t|�f � d S )Nz%d,%d-%d,%d: %s %s)�print�tok_name�repr) �typer Zxxx_todo_changemeZxxx_todo_changeme1�lineZsrowZscolZerowZecolr r r �
printtoken� s
�rF c C s( zt | |� W n tk
r" Y nX d S r8 )�
tokenize_loopr@ )�readline�
tokeneaterr r r r
� s
c C s t | �D ] }||� qd S r8 )r )rH rI Z
token_infor r r rG � s rG c @ s, e Zd Zdd� Zdd� Zdd� Zdd� Zd S )
� Untokenizerc C s g | _ d| _d| _d S )Nr r )�tokens�prev_row�prev_col)�selfr r r �__init__� s zUntokenizer.__init__c C s* |\}}|| j }|r&| j�d| � d S )N� )rM rK �append)rN �start�row�col�
col_offsetr r r �add_whitespace� s
zUntokenizer.add_whitespacec C s� |D ]p}t |�dkr$| �||� qv|\}}}}}| �|� | j�|� |\| _| _|ttfkr| jd7 _d| _qd� | j�S )N� r r r )
�len�compatrV rK rQ rL rM �NEWLINE�NLr )rN �iterable�t�tok_typer rR �endrE r r r r � s
zUntokenizer.untokenizec C s� d}g }| j j}|\}}|ttfkr,|d7 }|ttfkr<d}|D ]�}|d d� \}}|ttttfkrl|d7 }|tkr�|�|� q@n>|t kr�|�
� q@n*|ttfkr�d}n|r�|r�||d � d}||� q@d S )NFrP TrW ���) rK rQ �NAME�NUMBERrZ r[ �ASYNC�AWAIT�INDENT�DEDENT�pop) rN r r\ � startline�indents� toks_append�toknum�tokval�tokr r r rY � s0
zUntokenizer.compatN)r= r> r? rO rV r rY r r r r rJ � s rJ z&^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)s ^[ \t\f]*(?:[#\r\n]|$)c C sH | d d� � � �dd�}|dks*|�d�r.dS |dks@|�d�rDdS | S ) N� r �-�utf-8zutf-8-)zlatin-1�
iso-8859-1z iso-latin-1)zlatin-1-z iso-8859-1-z iso-latin-1-rq )�lower�replace�
startswith)�orig_enc�encr r r �_get_normal_name� s �rw c s� d� d }d}�fdd�}� fdd�}|� }|� t�rHd� |dd � }d }|sT|g fS ||�}|rj||gfS t�|�s~||gfS |� }|s�||gfS ||�}|r�|||gfS |||gfS )
NFrp c s( z� � W S t k
r" t� Y S X d S r8 )�
StopIteration�bytesr )rH r r � read_or_stop s z%detect_encoding.<locals>.read_or_stopc s� z| � d�}W n tk
r$ Y d S X t�|�}|s8d S t|�d��}z t|�}W n tk
rr td| ��Y nX � r�|j dkr�td��|d7 }|S )N�asciir zunknown encoding: rp zencoding problem: utf-8z-sig)
�decode�UnicodeDecodeError� cookie_re�matchrw r r � LookupError� SyntaxError�name)rE � line_stringr �encoding�codec)� bom_foundr r � find_cookie s"
z$detect_encoding.<locals>.find_cookieT� z utf-8-sig)rt r �blank_rer )rH r� �defaultrz r� �first�secondr )r� rH r �detect_encoding� s0
r� c C s t � }|�| �S r8 )rJ r )r\ �utr r r r : s c c st d } }}d\}}d }dg}d }d} d}
d} z
| � } W n t k
rP d} Y nX |d }dt| � }
}|�r2| s|td|��|�| �}|r�|�d� }
}t|| d |� |||f|| fV d\}}d }nd|�r | dd � dk�r | d d � d
k�r t|| ||t| �f|fV d}d }q.n|| }|| }q.�nB|dk�r\|�s\| �sL�q d}|
|k �r�| |
d k�rr|d }n8| |
d k�r�|t d t }n| |
d
k�r�d}n�q�|
d }
�qP|
|k�rĐq |�r�|V d }| |
dk�r�| |
dk�rT| |
d � �d�}|
t|� }t |||
f||
t|� f| fV t
| |d � ||f|t| �f| fV q.t
t f| |
dk | |
d � ||
f|t| �f| fV q.||d k�r�|� |� t | d |
� |df||
f| fV ||d k �r4||k�r�t
dd||
| f��|d d� }| �r|
|d k�rd} d} d}
td||
f||
f| fV �q�| �rt| �rt|
|d k�rtd} d} d}
n| �sptd|df��d}|
|k r.t�| |
�}|�r�|�d�\}}||f||f| }}}
| ||� | | }}|tjk�s�|dk�r�|dk�r�t|||| fV �q|dk�rJt} |dk�r t
} n
| �r&d} |�r6|V d }| |||| fV �q|dk�rx|�rd|V d }t |||| fV �q|tk�r�t| }|�| |
�}|�r�|�d�}
| ||
� }|�r�|V d }t||||
f| fV n||f}| |d � }| }q.�q|tk�s$|d d� tk�s$|d d� tk�r�|d dk�rx||f}t| �p\t|d �p\t|d }| |d � d }}| }q.n |�r�|V d }t|||| fV �q|�� �rr|dk�r�| �r�|dk�r�tnt|||| fV �qtt|||| f} |dk�r�|�s�| }�qt|d k�rZ|�rZ|d tk�rZ|d dk�rZd} |d }
t|d |d |d |d fV d }|�rj|V d }| V nz|d k�r�|�r�|V d }t
||||
f| fV d}nF|d k�r�|d }n|d k�r�|d }|�r�|V d }t |||| fV n(t| |
||
f||
d f| fV |
d }
�qtq.|�r,|V d }|dd � D ] } td|df|dfdfV �q8t d|df|dfdfV d S )!Nr )r r Fr r zEOF in multi-line string���z\
���z\
rP � � z#
�#z
r` z3unindent does not match any outer indentation levelz
<tokenize>zEOF in multi-line statement�.TrW r� �
)�async�awaitr� �def� �\z([{z)]}) rx rX r; r r_ �STRING�
ERRORTOKEN�tabsize�rstrip�COMMENTr[ rQ re �IndentationErrorrf �
pseudoprog�span�stringZdigitsrb rZ �
triple_quoted�endprogs�
single_quoted� isidentifierrc rd ra �OP� ENDMARKER) rH �lnum�parenlev� continued�contstr�needcont�contlineri ZstashedZ async_defZasync_def_indentZ async_def_nlrE �pos�max�strstart�endprog�endmatchr_ �column�
comment_tokenZnl_pos� pseudomatchrR �spos�eposr �initial�newlinerm �indentr r r r O s�
�*
�
�
�
�
� ��
��
�
�
�
� �
� �__main__)R�
__author__� __credits__r� �re�codecsr r Zlib2to3.pgen2.tokenr r �dir�__all__ry � NameError�strr r r r# �
Whitespace�Comment�Ignore�Name� Binnumber� Hexnumber� Octnumber� Decnumber� Intnumber�Exponent�
Pointfloat�Expfloat� Floatnumber�
Imagnumber�Number�Single�Double�Single3�Double3Z
_litprefix�Triple�StringZOperatorZBracket�Special�Funny�
PlainToken�Token�ContStr� PseudoExtras� PseudoToken�map�compileZ tokenprogr� r2 r7 Z _strprefixesr� r� r� r� � Exceptionr; r@ rF r
rG rJ �ASCIIr~ r� rw r� r r r= �sysrX �argv�openrH �stdinr r r r �<module> s� � �
� � ���
� ��� � � � �� � �� 8 I b
__pycache__/grammar.cpython-38.pyc 0000644 00000013043 15052147326 0013040 0 ustar 00 U
e5d� � @ s` d Z ddlZddlmZ G dd� de�ZdZi Ze�� D ]"Z e r8e �
� \Z Z e
ee �ee < q8dS )a� This module defines the data structures used to represent a grammar.
These are a bit arcane because they are derived from the data
structures used by Python's 'pgen' parser generator.
There's also a table here mapping operators to their names in the
token module; the Python tokenize module reports all operators as the
fallback token code OP, but the parser needs the actual token code.
� N� )�tokenc @ s@ e Zd ZdZdd� Zdd� Zdd� Zdd � Zd
d � Zd d
� Z dS )�Grammara� Pgen parsing tables conversion class.
Once initialized, this class supplies the grammar tables for the
parsing engine implemented by parse.py. The parsing engine
accesses the instance variables directly. The class here does not
provide initialization of the tables; several subclasses exist to
do this (see the conv and pgen modules).
The load() method reads the tables from a pickle file, which is
much faster than the other ways offered by subclasses. The pickle
file is written by calling dump() (after loading the grammar
tables using a subclass). The report() method prints a readable
representation of the tables to stdout, for debugging.
The instance variables are as follows:
symbol2number -- a dict mapping symbol names to numbers. Symbol
numbers are always 256 or higher, to distinguish
them from token numbers, which are between 0 and
255 (inclusive).
number2symbol -- a dict mapping numbers to symbol names;
these two are each other's inverse.
states -- a list of DFAs, where each DFA is a list of
states, each state is a list of arcs, and each
arc is a (i, j) pair where i is a label and j is
a state number. The DFA number is the index into
this list. (This name is slightly confusing.)
Final states are represented by a special arc of
the form (0, j) where j is its own state number.
dfas -- a dict mapping symbol numbers to (DFA, first)
pairs, where DFA is an item from the states list
above, and first is a set of tokens that can
begin this grammar rule (represented by a dict
whose values are always 1).
labels -- a list of (x, y) pairs where x is either a token
number or a symbol number, and y is either None
or a string; the strings are keywords. The label
number is the index in this list; label numbers
are used to mark state transitions (arcs) in the
DFAs.
start -- the number of the grammar's start symbol.
keywords -- a dict mapping keyword strings to arc labels.
tokens -- a dict mapping token numbers to arc labels.
c C s< i | _ i | _g | _i | _dg| _i | _i | _i | _d| _d S )N)r ZEMPTY� ) �
symbol2number�
number2symbol�states�dfas�labels�keywords�tokens� symbol2label�start)�self� r �-/usr/lib64/python3.8/lib2to3/pgen2/grammar.py�__init__L s zGrammar.__init__c C s, t |d��}t�| j|tj� W 5 Q R X dS )z)Dump the grammar tables to a pickle file.�wbN)�open�pickle�dump�__dict__ZHIGHEST_PROTOCOL)r �filename�fr r r r W s z Grammar.dumpc C s0 t |d��}t�|�}W 5 Q R X | j�|� dS )z+Load the grammar tables from a pickle file.�rbN)r r �loadr �update)r r r �dr r r r \ s z Grammar.loadc C s | j �t�|�� dS )z3Load the grammar tables from a pickle bytes object.N)r r r �loads)r Zpklr r r r b s z
Grammar.loadsc C sT | � � }dD ]}t||t| |��� � q | jdd� |_| jdd� |_| j|_|S )z#
Copy the grammar.
)r r r r r r
N)� __class__�setattr�getattr�copyr
r r )r �newZ dict_attrr r r r" f s z Grammar.copyc C sv ddl m } td� || j� td� || j� td� || j� td� || j� td� || j� td| j� d S )
z:Dump the grammar tables to standard output, for debugging.r )�pprintZs2nZn2sr r r
r N)r$ �printr r r r r
r )r r$ r r r �reports s
zGrammar.reportN)
�__name__�
__module__� __qualname__�__doc__r r r r r" r& r r r r r s 5
r a
( LPAR
) RPAR
[ LSQB
] RSQB
: COLON
, COMMA
; SEMI
+ PLUS
- MINUS
* STAR
/ SLASH
| VBAR
& AMPER
< LESS
> GREATER
= EQUAL
. DOT
% PERCENT
` BACKQUOTE
{ LBRACE
} RBRACE
@ AT
@= ATEQUAL
== EQEQUAL
!= NOTEQUAL
<> NOTEQUAL
<= LESSEQUAL
>= GREATEREQUAL
~ TILDE
^ CIRCUMFLEX
<< LEFTSHIFT
>> RIGHTSHIFT
** DOUBLESTAR
+= PLUSEQUAL
-= MINEQUAL
*= STAREQUAL
/= SLASHEQUAL
%= PERCENTEQUAL
&= AMPEREQUAL
|= VBAREQUAL
^= CIRCUMFLEXEQUAL
<<= LEFTSHIFTEQUAL
>>= RIGHTSHIFTEQUAL
**= DOUBLESTAREQUAL
// DOUBLESLASH
//= DOUBLESLASHEQUAL
-> RARROW
:= COLONEQUAL
)r* r � r �objectr Z opmap_rawZopmap�
splitlines�line�split�op�namer! r r r r �<module> s o3 __pycache__/__init__.cpython-38.opt-2.pyc 0000644 00000000213 15052147326 0014104 0 ustar 00 U
e5d� � @ s d S )N� r r r �./usr/lib64/python3.8/lib2to3/pgen2/__init__.py�<module> s __pycache__/literals.cpython-38.opt-2.pyc 0000644 00000002413 15052147326 0014170 0 ustar 00 U
e5dc � @ sL d dl Z dddddddd d
d d �
Zd
d� Zdd� Zdd� ZedkrHe� dS )� N��� �
�
� � �'�"�\)
�a�b�f�n�r�t�vr r
r c C s� | � dd�\}}t�|�}|d k r&|S |�d�r�|dd � }t|�dk rTtd| ��zt|d�}W q� tk
r� td| �d �Y q�X n2zt|d�}W n" tk
r� td| �d �Y nX t|�S ) Nr � �x� z!invalid hex string escape ('\%s')� � z#invalid octal string escape ('\%s'))�group�simple_escapes�get�
startswith�len�
ValueError�int�chr)�m�all�tailZescZhexes�i� r$ �./usr/lib64/python3.8/lib2to3/pgen2/literals.py�escape s"
r& c C sH | d }| d d� |d kr$|d }| t |�t |� � } t�dt| �S )Nr � z)\\(\'|\"|\\|[abfnrtv]|x.{0,2}|[0-7]{1,3}))r �re�subr& )�s�qr$ r$ r% �
evalString( s
r, c C s@ t d�D ]2} t| �}t|�}t|�}||krt| |||� qd S )N� )�ranger �reprr, �print)r# �cr* �er$ r$ r% �test2 s r3 �__main__)r( r r& r, r3 �__name__r$ r$ r$ r% �<module> s �
__pycache__/driver.cpython-38.opt-1.pyc 0000644 00000011747 15052147326 0013655 0 ustar 00 U
e5dQ � @ s� d Z dZddgZddlZddlZddlZddlZddlZddlm Z m
Z
m Z m Z m
Z
G dd� de�Zd d
� Zddd�Zdd� Zdd� Zdd� Zedkr�e�ee� �� dS )zZParser driver.
This provides a high-level interface to parse a file into a syntax tree.
z#Guido van Rossum <guido@python.org>�Driver� load_grammar� N� )�grammar�parse�token�tokenize�pgenc @ sH e Zd Zddd�Zddd�Zddd�Zdd d
�Zdd d �Zdd
d�ZdS )r Nc C s& || _ |d krt�� }|| _|| _d S )N)r �logging� getLogger�logger�convert)�selfr r
r � r �,/usr/lib64/python3.8/lib2to3/pgen2/driver.py�__init__ s
zDriver.__init__Fc C st t �| j| j�}|�� d}d}d } } } } }
d} |D �]} | \}}}} }
|||fkr�|\}
}||
k r�| d|
| 7 } |
}d}||k r�| |
||� 7 } |}|tjtjfkr�| |7 } | \}}|�d�r<|d7 }d}q<|t j
kr�tj | }|�r
| j �
dt j| || � |�||| |f��r6|�r0| j �
d� �qnd} | \}}|�d�r<|d7 }d}q<t �d||| |f��|jS ) z4Parse a series of tokens and return the syntax tree.r r N� �
z%s %r (prefix=%r)zStop.zincomplete input)r ZParserr r
Zsetupr �COMMENT�NL�endswithr �OPZopmapr �debug�tok_nameZaddtokenZ
ParseErrorZrootnode)r �tokensr �p�lineno�column�type�value�start�endZ line_text�prefixZ quintupleZs_linenoZs_columnr r r � parse_tokens& s^
�
�zDriver.parse_tokensc C s t �|j�}| �||�S �z*Parse a stream and return the syntax tree.)r �generate_tokens�readliner# )r �streamr r r r r �parse_stream_rawV s zDriver.parse_stream_rawc C s | � ||�S r$ )r( )r r' r r r r � parse_stream[ s zDriver.parse_streamc
C s4 t j|d|d��}| �||�W 5 Q R � S Q R X dS )z(Parse a file and return the syntax tree.�r)�encodingN)�io�openr) )r �filenamer+ r r' r r r �
parse_file_ s zDriver.parse_filec C s t �t�|�j�}| �||�S )z*Parse a string and return the syntax tree.)r r% r, �StringIOr& r# )r �textr r r r r � parse_stringd s zDriver.parse_string)NN)F)F)F)NF)F) �__name__�
__module__� __qualname__r r# r( r) r/ r2 r r r r r s
0
c C s: t j�| �\}}|dkr d}|| d�tttj�� d S )Nz.txtr �.z.pickle)�os�path�splitext�join�map�str�sys� version_info)�gt�head�tailr r r �_generate_pickle_namej s rB � Grammar.txtTFc
C s� |dkrt �� }|dkr t| �n|}|s2t|| �s�|�d| � t�| �}|r�|�d|� z|�|� W q� tk
r� } z|�d|� W 5 d}~X Y q�X nt �
� }|� |� |S )z'Load the grammar (maybe from a pickle).Nz!Generating grammar tables from %sz Writing grammar tables to %szWriting failed: %s) r
r rB �_newer�infor Zgenerate_grammar�dump�OSErrorr �Grammar�load)r? Zgp�save�forcer �g�er r r r q s
c C s8 t j�| �sdS t j�|�s dS t j�| �t j�|�kS )z0Inquire whether file a was written since file b.FT)r7 r8 �exists�getmtime)�a�br r r rD � s
rD c C sF t j�|�rt|�S tt j�|��}t�| |�}t� � }|�
|� |S )a� Normally, loads a pickled grammar by doing
pkgutil.get_data(package, pickled_grammar)
where *pickled_grammar* is computed from *grammar_source* by adding the
Python version and using a ``.pickle`` extension.
However, if *grammar_source* is an extant file, load_grammar(grammar_source)
is called instead. This facilitates using a packaged grammar file when needed
but preserves load_grammar's automatic regeneration behavior when possible.
) r7 r8 �isfiler rB �basename�pkgutil�get_datar rH �loads)�packageZgrammar_sourceZ pickled_name�datarL r r r �load_packaged_grammar� s
rY c G sB | st jdd� } tjtjt jdd� | D ]}t|ddd� q*dS )z�Main program, when run as a script: produce grammar pickle files.
Calls load_grammar for each argument, a path to a grammar text file.
r Nz %(message)s)�levelr' �formatT)rJ rK )r= �argvr
Z basicConfig�INFO�stdoutr )�argsr? r r r �main� s �r` �__main__)rC NTFN)�__doc__�
__author__�__all__r, r7 r
rT r= r r r r r r �objectr rB r rD rY r` r3 �exit�intr r r r �<module> s( M �
__pycache__/literals.cpython-38.pyc 0000644 00000003024 15052147326 0013227 0 ustar 00 U
e5dc � @ sP d Z ddlZddddddd d
d d d
�
Zdd� Zdd� Zdd� ZedkrLe� dS )z<Safely evaluate Python string literals without using eval().� N��� �
�
� � �'�"�\)
�a�b�f�n�r�t�vr r
r c C s� | � dd�\}}|�d�s t�t�|�}|d k r4|S |�d�r�|dd � }t|�dk rbtd| ��zt|d�}W q� tk
r� td| �d �Y q�X n2zt|d�}W n" tk
r� td | �d �Y nX t|�S )
Nr � r �x� z!invalid hex string escape ('\%s')� � z#invalid octal string escape ('\%s')) �group�
startswith�AssertionError�simple_escapes�get�len�
ValueError�int�chr)�m�all�tailZescZhexes�i� r% �./usr/lib64/python3.8/lib2to3/pgen2/literals.py�escape s$
r' c C s� | � d�s(| � d�s(tt| d d� ���| d }| d d� |d krL|d }| �|�sptt| t|� d � ���t| �dt|� ks�t�| t|�t|� � } t�dt| �S )Nr r
r r � r z)\\(\'|\"|\\|[abfnrtv]|x.{0,2}|[0-7]{1,3}))r r �repr�endswithr �re�subr' )�s�qr% r% r& �
evalString( s ($r/ c C s@ t d�D ]2} t| �}t|�}t|�}||krt| |||� qd S )N� )�ranger r) r/ �print)r$ �cr- �er% r% r& �test2 s r5 �__main__)�__doc__r+ r r' r/ r5 �__name__r% r% r% r& �<module> s" �
__pycache__/parse.cpython-38.opt-1.pyc 0000644 00000014506 15052147326 0013470 0 ustar 00 U
e5d� � @ s4 d Z ddlmZ G dd� de�ZG dd� de�ZdS )z�Parser engine for the grammar tables generated by pgen.
The grammar table must be loaded first.
See Parser/parser.c in the Python distribution for additional info on
how this parsing engine works.
� )�tokenc @ s e Zd ZdZdd� Zdd� ZdS )�
ParseErrorz(Exception to signal the parser is stuck.c C s4 t �| d||||f � || _|| _|| _|| _d S )Nz!%s: type=%r, value=%r, context=%r)� Exception�__init__�msg�type�value�context)�selfr r r r � r �+/usr/lib64/python3.8/lib2to3/pgen2/parse.pyr s
�zParseError.__init__c C s t | �| j| j | j| jffS �N)r r r r )r
r r r �
__reduce__ s zParseError.__reduce__N)�__name__�
__module__� __qualname__�__doc__r r r r r r r s r c @ sL e Zd ZdZddd�Zddd�Zdd� Zd d
� Zd d � Zd
d� Z dd� Z
dS )�Parsera5 Parser engine.
The proper usage sequence is:
p = Parser(grammar, [converter]) # create instance
p.setup([start]) # prepare for parsing
<for each input token>:
if p.addtoken(...): # parse a token; may raise ParseError
break
root = p.rootnode # root of abstract syntax tree
A Parser instance may be reused by calling setup() repeatedly.
A Parser instance contains state pertaining to the current token
sequence, and should not be used concurrently by different threads
to parse separate token sequences.
See driver.py for how to get input tokens by tokenizing a file or
string.
Parsing is complete when addtoken() returns True; the root of the
abstract syntax tree can then be retrieved from the rootnode
instance variable. When a syntax error occurs, addtoken() raises
the ParseError exception. There is no error recovery; the parser
cannot be used after a syntax error was reported (but it can be
reinitialized by calling setup()).
Nc C s || _ |pdd� | _dS )a� Constructor.
The grammar argument is a grammar.Grammar instance; see the
grammar module for more information.
The parser is not ready yet for parsing; you must call the
setup() method to get it started.
The optional convert argument is a function mapping concrete
syntax tree nodes to abstract syntax tree nodes. If not
given, no conversion is done and the syntax tree produced is
the concrete syntax tree. If given, it must be a function of
two arguments, the first being the grammar (a grammar.Grammar
instance), and the second being the concrete syntax tree node
to be converted. The syntax tree is converted from the bottom
up.
A concrete syntax tree node is a (type, value, context, nodes)
tuple, where type is the node type (a token or symbol number),
value is None for symbols and a string for tokens, context is
None or an opaque value used for error reporting (typically a
(lineno, offset) pair), and nodes is a list of children for
symbols, and None for tokens.
An abstract syntax tree node may be anything; this is entirely
up to the converter function.
c S s |S r
r )�grammar�noder r r �<lambda>Z � z!Parser.__init__.<locals>.<lambda>N)r �convert)r
r r r r r r < s zParser.__init__c C sH |dkr| j j}|ddg f}| j j| d|f}|g| _d| _t� | _dS )a� Prepare for parsing.
This *must* be called before starting to parse.
The optional argument is an alternative start symbol; it
defaults to the grammar's start symbol.
You can use a Parser instance to parse any number of programs;
each time you call setup() the parser is reset to an initial
state determined by the (implicit or explicit) start symbol.
N� )r �start�dfas�stack�rootnode�set�
used_names)r
r �newnodeZ
stackentryr r r �setup\ s
z Parser.setupc C s0 | � |||�}| jd \}}}|\}} || }
|
D ]�\} } | jj| \}
}|| kr�| �||| |� | }|| d|fgkr�| �� | js� dS | jd \}}}|\}} qf dS |
dkr2| jj|
}|\}}||kr2| �|
| jj|
| |� qq2d|f|
k�r | �� | j�s*td|||��qtd|||��qdS ) z<Add a token; return True iff this is the end of the program.���r TF� ztoo much inputz bad inputN) �classifyr r �labels�shift�popr �pushr )r
r r r �ilabel�dfa�stater Zstates�firstZarcs�i�newstate�t�vZitsdfaZ itsstatesZitsfirstr r r �addtokent s>
�zParser.addtokenc C sX |t jkr0| j�|� | jj�|�}|dk r0|S | jj�|�}|dkrTtd|||��|S )z&Turn a token into a label. (Internal)Nz bad token) r �NAMEr �addr �keywords�get�tokensr )r
r r r r) r r r r$ � s
zParser.classifyc C sT | j d \}}}|||df}| �| j|�}|dk r@|d �|� |||f| j d< dS )zShift a token. (Internal)r" N)r r r �append) r
r r r. r r* r+ r r r r r r& � s z Parser.shiftc C sB | j d \}}}|d|g f}|||f| j d< | j �|d|f� dS )z Push a nonterminal. (Internal)r" Nr )r r7 ) r
r Znewdfar. r r* r+ r r r r r r( � s z Parser.pushc C s` | j �� \}}}| �| j|�}|dk r\| j rL| j d \}}}|d �|� n|| _| j| j_dS )z Pop a nonterminal. (Internal)Nr" )r r' r r r7 r r )r
ZpopdfaZpopstateZpopnoder r* r+ r r r r r' � s z
Parser.pop)N)N) r r r r r r! r1 r$ r& r( r' r r r r r s
0 r N)r � r r r �objectr r r r r �<module> s
__pycache__/token.cpython-38.pyc 0000644 00000003541 15052147326 0012534 0 ustar 00 U
e5d � @ sP d Z dZdZdZdZdZdZdZdZd Z d
Z
d Z d Z d
Z
dZdZdZdZdZdZdZdZdZdZdZdZdZdZd Z d Z d Z d Z d Z d!Z!d"Z"d#Z#d$Z$d%Z%d&Z&d'Z'd(Z(d)Z)d*Z*d+Z+d,Z,d-Z-d.Z.d/Z/d0Z0d1Z1d2Z2d3Z3d4Z4d5Z5d6Z6d7Z7d8Z8d9Z9d:Z:d;Z;d<Z<d=Z=d>Z>i Z?e@eA� �B� �D ]$\ZCZDeEeD�eEd�k�reCe?eD< �qd?d@� ZFdAdB� ZGdCdD� ZHdES )Fz!Token constants (from "token.h").� � � � � � � � � � �
� � �
� � � � � � � � � � � � � � � � � � � �! �" �# �$ �% �& �' �( �) �* �+ �, �- �. �/ �0 �1 �2 �3 �4 �5 �6 �7 �8 �9 �: �; �< � c C s | t k S �N�� NT_OFFSET��x� rD �+/usr/lib64/python3.8/lib2to3/pgen2/token.py�
ISTERMINALO s rF c C s | t kS r? r@ rB rD rD rE �
ISNONTERMINALR s rG c C s | t kS r? )� ENDMARKERrB rD rD rE �ISEOFU s rI N)I�__doc__rH �NAME�NUMBER�STRING�NEWLINE�INDENT�DEDENT�LPAR�RPAR�LSQB�RSQB�COLON�COMMA�SEMI�PLUS�MINUS�STAR�SLASH�VBAR�AMPER�LESS�GREATER�EQUAL�DOT�PERCENTZ BACKQUOTE�LBRACE�RBRACE�EQEQUAL�NOTEQUAL� LESSEQUAL� GREATEREQUAL�TILDE�
CIRCUMFLEX� LEFTSHIFT�
RIGHTSHIFT�
DOUBLESTAR� PLUSEQUAL�MINEQUAL� STAREQUAL�
SLASHEQUAL� PERCENTEQUAL�
AMPEREQUAL� VBAREQUAL�CIRCUMFLEXEQUAL�LEFTSHIFTEQUAL�RIGHTSHIFTEQUAL�DOUBLESTAREQUAL� DOUBLESLASH�DOUBLESLASHEQUAL�AT�ATEQUAL�OP�COMMENT�NL�RARROW�AWAIT�ASYNC�
ERRORTOKEN�
COLONEQUAL�N_TOKENSrA �tok_name�list�globals�items�_nameZ_value�typerF rG rI rD rD rD rE �<module> s� __pycache__/conv.cpython-38.opt-2.pyc 0000644 00000007112 15052147326 0013317 0 ustar 00 U
e5d�% � @ s. d dl Z d dlmZmZ G dd� dej�ZdS )� N)�grammar�tokenc @ s, e Zd Zdd� Zdd� Zdd� Zdd� Zd S )
� Converterc C s | � |� | �|� | �� d S �N)�parse_graminit_h�parse_graminit_c�
finish_off)�selfZ
graminit_hZ
graminit_c� r
�*/usr/lib64/python3.8/lib2to3/pgen2/conv.py�run/ s
z
Converter.runc
C s� z t |�}W n8 tk
rD } ztd||f � W Y �dS d }~X Y nX i | _i | _d}|D ]d}|d7 }t�d|�}|s�|�� r�td|||�� f � qZ|�� \}}t |�}|| j|< || j|< qZdS )N�Can't open %s: %sFr � z^#define\s+(\w+)\s+(\d+)$z%s(%s): can't parse %sT)
�open�OSError�printZ
symbol2numberZ
number2symbol�re�match�strip�groups�int) r �filename�f�err�lineno�line�mo�symbol�numberr
r
r r 5 s( �
zConverter.parse_graminit_hc !
C s� z t |�}W n8 tk
rD } ztd||f � W Y �dS d }~X Y nX d}|d t|� }}|d t|� }}|d t|� }}i }g }|�d��r�|�d��rJt�d|�}ttt |�
� ��\} }
} g } t | �D ]F}
|d t|� }}t�d|�}ttt |�
� ��\}}| � ||f� q�|d t|� }}| || |
f< |d t|� }}q�t�d|�}ttt |�
� ��\}}g }t |�D ]R}
|d t|� }}t�d |�}ttt |�
� ��\} } }
|| |
f } |� | � �qx|� |� |d t|� }}|d t|� }}q�|| _
i }t�d
|�}t |�d��}t |�D ]�}|d t|� }}t�d |�}|�d �}ttt |�dd
dd���\}}}}|| }|d t|� }}t�d|�}i }t|�d��}t|�D ]@\}} t| �} t d�D ]$}| d|> @ �r�d||d | < �qΐq�||f||< �q(|d t|� }}|| _g } |d t|� }}t�d|�}t |�d��} t | �D ]^}|d t|� }}t�d|�}|�
� \}}t |�}|dk�r�d }nt|�}| � ||f� �qX|d t|� }}| | _|d t|� }}|d t|� }}t�d|�}t |�d��}|d t|� }}|d t|� }}t�d|�}t |�d��} |d t|� }}t�d|�}t |�d��} | | _|d t|� }}z|d t|� }}W n tk
�r� Y nX d S )Nr
Fr r z static arc z)static arc arcs_(\d+)_(\d+)\[(\d+)\] = {$z\s+{(\d+), (\d+)},$z'static state states_(\d+)\[(\d+)\] = {$z \s+{(\d+), arcs_(\d+)_(\d+)},$z static dfa dfas\[(\d+)\] = {$z0\s+{(\d+), "(\w+)", (\d+), (\d+), states_(\d+),$� � � � z\s+("(?:\\\d\d\d)*")},$� z!static label labels\[(\d+)\] = {$z\s+{(\d+), (0|"\w+")},$�0z
\s+(\d+),$z\s+{(\d+), labels},$z \s+(\d+)$)r r r �next�
startswithr r �list�mapr r �range�append�states�group�eval� enumerate�ord�dfas�labels�start�
StopIteration)!r r r r r r Zallarcsr+ r �n�m�kZarcs�_�i�j�s�t�stater0 Zndfasr r �x�y�z�firstZ rawbitset�cZbyter1 Znlabelsr2 r
r
r r T s� �
�
"
zConverter.parse_graminit_cc C sX i | _ i | _t| j�D ]<\}\}}|tjkr@|d k r@|| j |< q|d kr|| j|< qd S r )�keywords�tokensr. r1 r �NAME)r Zilabel�type�valuer
r
r r � s zConverter.finish_offN)�__name__�
__module__� __qualname__r r r r r
r
r
r r $ s
&r )r Zpgen2r r ZGrammarr r
r
r
r �<module> s __pycache__/driver.cpython-38.opt-2.pyc 0000644 00000010003 15052147326 0013636 0 ustar 00 U
e5dQ � @ s� d Z ddgZddlZddlZddlZddlZddlZddlmZm Z m
Z
m Z m Z G dd� de
�Zdd � Zdd
d�Zdd� Zdd� Zdd� Zedkr�e�ee� �� dS )z#Guido van Rossum <guido@python.org>�Driver� load_grammar� N� )�grammar�parse�token�tokenize�pgenc @ sH e Zd Zddd�Zddd�Zddd�Zdd d
�Zdd d �Zdd
d�ZdS )r Nc C s& || _ |d krt�� }|| _|| _d S �N)r �logging� getLogger�logger�convert)�selfr r r
� r �,/usr/lib64/python3.8/lib2to3/pgen2/driver.py�__init__ s
zDriver.__init__Fc C st t �| j| j�}|�� d}d}d } } } } }
d} |D �]} | \}}}} }
|||fkr�|\}
}||
k r�| d|
| 7 } |
}d}||k r�| |
||� 7 } |}|tjtjfkr�| |7 } | \}}|�d�r<|d7 }d}q<|t j
kr�tj | }|�r
| j �
dt j| || � |�||| |f��r6|�r0| j �
d� �qnd} | \}}|�d�r<|d7 }d}q<t �d||| |f��|jS )Nr r � �
z%s %r (prefix=%r)zStop.zincomplete input)r ZParserr r Zsetupr �COMMENT�NL�endswithr �OPZopmapr
�debug�tok_nameZaddtokenZ
ParseErrorZrootnode)r �tokensr �p�lineno�column�type�value�start�endZ line_text�prefixZ quintupleZs_linenoZs_columnr r r � parse_tokens& s^
�
�zDriver.parse_tokensc C s t �|j�}| �||�S r
)r �generate_tokens�readliner$ )r �streamr r r r r �parse_stream_rawV s zDriver.parse_stream_rawc C s | � ||�S r
)r( )r r' r r r r � parse_stream[ s zDriver.parse_streamc
C s4 t j|d|d��}| �||�W 5 Q R � S Q R X d S )N�r)�encoding)�io�openr) )r �filenamer+ r r' r r r �
parse_file_ s zDriver.parse_filec C s t �t�|�j�}| �||�S r
)r r% r, �StringIOr& r$ )r �textr r r r r � parse_stringd s zDriver.parse_string)NN)F)F)F)NF)F) �__name__�
__module__� __qualname__r r$ r( r) r/ r2 r r r r r s
0
c C s: t j�| �\}}|dkr d}|| d�tttj�� d S )Nz.txtr �.z.pickle)�os�path�splitext�join�map�str�sys� version_info)�gt�head�tailr r r �_generate_pickle_namej s rB � Grammar.txtTFc
C s� |d krt �� }|d kr t| �n|}|s2t|| �s�|�d| � t�| �}|r�|�d|� z|�|� W q� tk
r� } z|�d|� W 5 d }~X Y q�X nt �
� }|� |� |S )Nz!Generating grammar tables from %sz Writing grammar tables to %szWriting failed: %s) r r rB �_newer�infor Zgenerate_grammar�dump�OSErrorr �Grammar�load)r? Zgp�save�forcer
�g�er r r r q s
c C s8 t j�| �sdS t j�|�s dS t j�| �t j�|�kS )NFT)r7 r8 �exists�getmtime)�a�br r r rD � s
rD c C sF t j�|�rt|�S tt j�|��}t�| |�}t� � }|�
|� |S r
) r7 r8 �isfiler rB �basename�pkgutil�get_datar rH �loads)�packageZgrammar_sourceZ pickled_name�datarL r r r �load_packaged_grammar� s
rY c G sB | st jdd � } tjtjt jdd� | D ]}t|ddd� q*dS )Nr z %(message)s)�levelr' �formatT)rJ rK )r= �argvr Z basicConfig�INFO�stdoutr )�argsr? r r r �main� s �r` �__main__)rC NTFN)�
__author__�__all__r, r7 r rT r= r r r r r r �objectr rB r rD rY r` r3 �exit�intr r r r �<module> s& M �
__pycache__/driver.cpython-38.pyc 0000644 00000012027 15052147326 0012706 0 ustar 00 U
e5dQ � @ s� d Z dZddgZddlZddlZddlZddlZddlZddlm Z m
Z
m Z m Z m
Z
G dd� de�Zd d
� Zddd�Zdd� Zdd� Zdd� Zedkr�e�ee� �� dS )zZParser driver.
This provides a high-level interface to parse a file into a syntax tree.
z#Guido van Rossum <guido@python.org>�Driver� load_grammar� N� )�grammar�parse�token�tokenize�pgenc @ sH e Zd Zddd�Zddd�Zddd�Zdd d
�Zdd d �Zdd
d�ZdS )r Nc C s& || _ |d krt�� }|| _|| _d S )N)r �logging� getLogger�logger�convert)�selfr r
r � r �,/usr/lib64/python3.8/lib2to3/pgen2/driver.py�__init__ s
zDriver.__init__Fc C s� t �| j| j�}|�� d}d}d } } } } }
d} |D �]8} | \}}}} }
|||fkr�||f|ksxt||f|f��|\}
}||
k r�| d|
| 7 } |
}d}||k r�| |
||� 7 } |}|tjtjfkr�| |7 } | \}}|� d�r<|d7 }d}q<|t
j k�r
tj | }|�r(| j
�dt
j| || � |�||| |f��rT|�rN| j
�d� �q�d} | \}}|� d�r<|d7 }d}q<t �d||| |f��|jS ) z4Parse a series of tokens and return the syntax tree.r r N� �
z%s %r (prefix=%r)zStop.zincomplete input)r ZParserr r
Zsetup�AssertionErrorr �COMMENT�NL�endswithr �OPZopmapr �debug�tok_nameZaddtokenZ
ParseErrorZrootnode)r �tokensr �p�lineno�column�type�value�start�endZ line_text�prefixZ quintupleZs_linenoZs_columnr r r � parse_tokens& s`
�
�zDriver.parse_tokensc C s t �|j�}| �||�S �z*Parse a stream and return the syntax tree.)r �generate_tokens�readliner$ )r �streamr r r r r �parse_stream_rawV s zDriver.parse_stream_rawc C s | � ||�S r% )r) )r r( r r r r � parse_stream[ s zDriver.parse_streamc
C s4 t j|d|d��}| �||�W 5 Q R � S Q R X dS )z(Parse a file and return the syntax tree.�r)�encodingN)�io�openr* )r �filenamer, r r( r r r �
parse_file_ s zDriver.parse_filec C s t �t�|�j�}| �||�S )z*Parse a string and return the syntax tree.)r r& r- �StringIOr' r$ )r �textr r r r r � parse_stringd s zDriver.parse_string)NN)F)F)F)NF)F) �__name__�
__module__� __qualname__r r$ r) r* r0 r3 r r r r r s
0
c C s: t j�| �\}}|dkr d}|| d�tttj�� d S )Nz.txtr �.z.pickle)�os�path�splitext�join�map�str�sys� version_info)�gt�head�tailr r r �_generate_pickle_namej s rC � Grammar.txtTFc
C s� |dkrt �� }|dkr t| �n|}|s2t|| �s�|�d| � t�| �}|r�|�d|� z|�|� W q� tk
r� } z|�d|� W 5 d}~X Y q�X nt �
� }|� |� |S )z'Load the grammar (maybe from a pickle).Nz!Generating grammar tables from %sz Writing grammar tables to %szWriting failed: %s) r
r rC �_newer�infor Zgenerate_grammar�dump�OSErrorr �Grammar�load)r@ Zgp�save�forcer �g�er r r r q s
c C s8 t j�| �sdS t j�|�s dS t j�| �t j�|�kS )z0Inquire whether file a was written since file b.FT)r8 r9 �exists�getmtime)�a�br r r rE � s
rE c C sF t j�|�rt|�S tt j�|��}t�| |�}t� � }|�
|� |S )a� Normally, loads a pickled grammar by doing
pkgutil.get_data(package, pickled_grammar)
where *pickled_grammar* is computed from *grammar_source* by adding the
Python version and using a ``.pickle`` extension.
However, if *grammar_source* is an extant file, load_grammar(grammar_source)
is called instead. This facilitates using a packaged grammar file when needed
but preserves load_grammar's automatic regeneration behavior when possible.
) r8 r9 �isfiler rC �basename�pkgutil�get_datar rI �loads)�packageZgrammar_sourceZ pickled_name�datarM r r r �load_packaged_grammar� s
rZ c G sB | st jdd� } tjtjt jdd� | D ]}t|ddd� q*dS )z�Main program, when run as a script: produce grammar pickle files.
Calls load_grammar for each argument, a path to a grammar text file.
r Nz %(message)s)�levelr( �formatT)rK rL )r> �argvr
Z basicConfig�INFO�stdoutr )�argsr@ r r r �main� s �ra �__main__)rD NTFN)�__doc__�
__author__�__all__r- r8 r
rU r> r r r r r r �objectr rC r rE rZ ra r4 �exit�intr r r r �<module> s( M �
__pycache__/literals.cpython-38.opt-1.pyc 0000644 00000002530 15052147326 0014167 0 ustar 00 U
e5dc � @ sP d Z ddlZddddddd d
d d d
�
Zdd� Zdd� Zdd� ZedkrLe� dS )z<Safely evaluate Python string literals without using eval().� N��� �
�
� � �'�"�\)
�a�b�f�n�r�t�vr r
r c C s� | � dd�\}}t�|�}|d k r&|S |�d�r�|dd � }t|�dk rTtd| ��zt|d�}W q� tk
r� td| �d �Y q�X n2zt|d�}W n" tk
r� td| �d �Y nX t|�S ) Nr � �x� z!invalid hex string escape ('\%s')� � z#invalid octal string escape ('\%s'))�group�simple_escapes�get�
startswith�len�
ValueError�int�chr)�m�all�tailZescZhexes�i� r$ �./usr/lib64/python3.8/lib2to3/pgen2/literals.py�escape s"
r& c C sH | d }| d d� |d kr$|d }| t |�t |� � } t�dt| �S )Nr � z)\\(\'|\"|\\|[abfnrtv]|x.{0,2}|[0-7]{1,3}))r �re�subr& )�s�qr$ r$ r% �
evalString( s
r, c C s@ t d�D ]2} t| �}t|�}t|�}||krt| |||� qd S )N� )�ranger �reprr, �print)r# �cr* �er$ r$ r% �test2 s r3 �__main__)�__doc__r( r r& r, r3 �__name__r$ r$ r$ r% �<module> s" �
__pycache__/conv.cpython-38.pyc 0000644 00000015556 15052147326 0012372 0 ustar 00 U
e5d�% � @ s2 d Z ddlZddlmZmZ G dd� dej�ZdS )a� Convert graminit.[ch] spit out by pgen to Python code.
Pgen is the Python parser generator. It is useful to quickly create a
parser from a grammar file in Python's grammar notation. But I don't
want my parsers to be written in C (yet), so I'm translating the
parsing tables to Python data structures and writing a Python parse
engine.
Note that the token numbers are constants determined by the standard
Python tokenizer. The standard token module defines these numbers and
their names (the names are not used much). The token numbers are
hardcoded into the Python tokenizer and into pgen. A Python
implementation of the Python tokenizer is also available, in the
standard tokenize module.
On the other hand, symbol numbers (representing the grammar's
non-terminals) are assigned by pgen based on the actual grammar
input.
Note: this module is pretty much obsolete; the pgen module generates
equivalent grammar tables directly from the Grammar.txt input file
without having to invoke the Python pgen C program.
� N)�grammar�tokenc @ s0 e Zd ZdZdd� Zdd� Zdd� Zdd � Zd
S ) � Convertera2 Grammar subclass that reads classic pgen output files.
The run() method reads the tables as produced by the pgen parser
generator, typically contained in two C files, graminit.h and
graminit.c. The other methods are for internal use only.
See the base class for more documentation.
c C s | � |� | �|� | �� dS )z<Load the grammar tables from the text files written by pgen.N)�parse_graminit_h�parse_graminit_c�
finish_off)�selfZ
graminit_hZ
graminit_c� r �*/usr/lib64/python3.8/lib2to3/pgen2/conv.py�run/ s
z
Converter.runc
C s� z t |�}W n8 tk
rD } ztd||f � W Y �dS d}~X Y nX i | _i | _d}|D ]�}|d7 }t�d|�}|s�|�� r�td|||�� f � qZ|�� \}}t |�}|| jks�t
�|| jks�t
�|| j|< || j|< qZdS ) z�Parse the .h file written by pgen. (Internal)
This file is a sequence of #define statements defining the
nonterminals of the grammar as numbers. We build two tables
mapping the numbers to names and back.
�Can't open %s: %sFNr � z^#define\s+(\w+)\s+(\d+)$z%s(%s): can't parse %sT) �open�OSError�print�
symbol2number�
number2symbol�re�match�strip�groups�int�AssertionError) r �filename�f�err�lineno�line�mo�symbol�numberr r r
r 5 s, �
zConverter.parse_graminit_hc !
C sT z t |�}W n8 tk
rD } ztd||f � W Y �dS d}~X Y nX d}|d t|� }}|dkspt||f��|d t|� }}|dks�t||f��|d t|� }}i }g }|�d��r�|�d��r�t�d |�}|s�t||f��tt t
|� � ��\} }
} g } t | �D ]Z}
|d t|� }}t�d
|�}|�s<t||f��tt t
|� � ��\}}| �
||f� �q|d t|� }}|d k�s�t||f��| || |
f< |d t|� }}q�t�d |�}|�s�t||f��tt t
|� � ��\}}|t|�k�s�t||f��g }t |�D ]~}
|d t|� }}t�d
|�}|�s:t||f��tt t
|� � ��\} } }
|| |
f } | t| �k�sxt||f��|�
| � �q|�
|� |d t|� }}|d k�s�t||f��|d t|� }}q�|| _i }t�d|�}|�s�t||f��t
|�d��}t |�D �]j}|d t|� }}t�d|�}|�s@t||f��|�d�}tt t
|�dddd���\}}}}| j| |k�s�t||f��| j| |k�s�t||f��|dk�s�t||f��|| }|t|�k�s�t||f��|d t|� }}t�d|�}|�s t||f��i }t|�d��}t|�D ]@\}} t| �} t d�D ]$}| d|> @ �r>d||d | < �q>�q&||f||< �q
|d t|� }}|d k�s�t||f��|| _g } |d t|� }}t�d|�}|�s�t||f��t
|�d��} t | �D ]p}|d t|� }}t�d|�}|�s$t||f��|� � \}}t
|�}|dk�rHd}nt|�}| �
||f� �q�|d t|� }}|d k�s�t||f��| | _|d t|� }}|dk�s�t||f��|d t|� }}t�d|�}|�s�t||f��t
|�d��}|t| j�k�s
t�|d t|� }}|dk�s2t||f��|d t|� }}t�d |�}|�sbt||f��t
|�d��} | t| j�k�s�t||f��|d t|� }}t�d |�}|�s�t||f��t
|�d��} | | jk�s�t||f��| | _|d t|� }}|d k�st||f��z|d t|� }}W n tk
�r< Y nX d�sPt||f��dS ) a� Parse the .c file written by pgen. (Internal)
The file looks as follows. The first two lines are always this:
#include "pgenheaders.h"
#include "grammar.h"
After that come four blocks:
1) one or more state definitions
2) a table defining dfas
3) a table defining labels
4) a struct defining the grammar
A state definition has the following form:
- one or more arc arrays, each of the form:
static arc arcs_<n>_<m>[<k>] = {
{<i>, <j>},
...
};
- followed by a state array, of the form:
static state states_<s>[<t>] = {
{<k>, arcs_<n>_<m>},
...
};
r FNr r
z#include "pgenheaders.h"
z#include "grammar.h"
z static arc z)static arc arcs_(\d+)_(\d+)\[(\d+)\] = {$z\s+{(\d+), (\d+)},$z};
z'static state states_(\d+)\[(\d+)\] = {$z \s+{(\d+), arcs_(\d+)_(\d+)},$z static dfa dfas\[(\d+)\] = {$z0\s+{(\d+), "(\w+)", (\d+), (\d+), states_(\d+),$� � � � z\s+("(?:\\\d\d\d)*")},$� z!static label labels\[(\d+)\] = {$z\s+{(\d+), (0|"\w+")},$�0z grammar _PyParser_Grammar = {
z
\s+(\d+),$z dfas,
z\s+{(\d+), labels},$z \s+(\d+)$)r r r �nextr �
startswithr r �list�mapr r �range�append�len�states�groupr r �eval� enumerate�ord�dfas�labels�start�
StopIteration)!r r r r r r Zallarcsr. r �n�m�kZarcs�_�i�j�s�t�stater3 Zndfasr r �x�y�z�firstZ rawbitset�cZbyter4 Znlabelsr5 r r r
r T s� �
�
"
zConverter.parse_graminit_cc C sX i | _ i | _t| j�D ]<\}\}}|tjkr@|dk r@|| j |< q|dkr|| j|< qdS )z1Create additional useful structures. (Internal).N)�keywords�tokensr1 r4 r �NAME)r Zilabel�type�valuer r r
r � s zConverter.finish_offN)�__name__�
__module__� __qualname__�__doc__r r r r r r r r
r $ s
&r )rM r Zpgen2r r ZGrammarr r r r r
�<module> s __pycache__/pgen.cpython-38.opt-2.pyc 0000644 00000022161 15052147326 0013304 0 ustar 00 U
e5d�5 � @ sd d dl mZmZmZ G dd� dej�ZG dd� de�ZG dd� de�ZG dd � d e�Z dd d �Z
d
S )� )�grammar�token�tokenizec @ s e Zd ZdS )� PgenGrammarN)�__name__�
__module__� __qualname__� r r �*/usr/lib64/python3.8/lib2to3/pgen2/pgen.pyr s r c @ s� e Zd Zd&dd�Zdd� Zdd� Zdd � Zd
d � Zd d
� Zdd� Z dd� Z
dd� Z dd� Z dd� Z
dd� Zdd� Zd d � Zd d � Zd'd d!�Zd"d#� Zd$d%� ZdS )(�ParserGeneratorNc C sl d }|d krt |�}|j}|| _|| _t�|j�| _| �� | � � \| _
| _ |d k rZ|� i | _ | �
� d S �N)�open�close�filename�streamr �generate_tokens�readline� generator�gettoken�parse�dfas� startsymbol�first� addfirstsets)�selfr r Z close_streamr r r
�__init__ s zParserGenerator.__init__c C s t � }t| j�� �}|�� |�| j� |�d| j� |D ]&}dt|j � }||j |< ||j
|< q:|D ]�}| j| }g }|D ]`}g }t |j �
� �D ]$\} }
|�| �|| �|�|
�f� q�|jr�|�d|�|�f� |�|� q||j�|� || �||�f|j|j | < qf|j | j |_|S )N� � )r �listr �keys�sort�remover �insert�len�
symbol2numberZ
number2symbol�sorted�arcs�items�append�
make_label�index�isfinal�states�
make_first�start) r �c�names�name�i�dfar, �stater& �label�nextr r r
� make_grammar s.
z ParserGenerator.make_grammarc C s4 | j | }i }t|�D ]}| �||�}d||< q|S �Nr )r r% r) )r r/ r1 Zrawfirstr r5 �ilabelr r r
r- 4 s
zParserGenerator.make_firstc C s& t |j�}|d �� r�||jkrZ||jkr4|j| S |j�|j| d f� ||j|< |S n>tt|d �}||jkrz|j| S |j�|d f� ||j|< |S n�t |�}|d �� r�||j
kr�|j
| S |j�tj |f� ||j
|< |S n>t j
| }||jk�r|j| S |j�|d f� ||j|< |S d S �Nr )r# �labels�isalphar$ Z symbol2labelr( �getattrr �tokens�eval�keywords�NAMEr Zopmap)r r/ r5 r9 Zitoken�valuer r r
r) = s6
zParserGenerator.make_labelc C s8 t | j�� �}|�� |D ]}|| jkr| �|� qd S r )r r r r r � calcfirst)r r0 r1 r r r
r k s
z ParserGenerator.addfirstsetsc C s� | j | }d | j|< |d }i }i }|j�� D ]x\}}|| j kr�|| jkrj| j| }|d kr~td| ��n| �|� | j| }|�|� |||< q.d||< |di||< q.i } |�� D ]:\}}
|
D ],} | | kr�td|| || | f ��|| | < q�q�|| j|< d S )Nr zrecursion for rule %rr zArule %s is ambiguous; %s is in the first sets of %s as well as %s)r r r& r' �
ValueErrorrC �update) r r1 r3 r4 ZtotalsetZ overlapcheckr5 r6 �fsetZinverseZitsfirstZsymbolr r r
rC s s4
� zParserGenerator.calcfirstc C s� i }d }| j tjkr�| j tjkr*| �� q| �tj�}| �tjd� | �� \}}| �tj� | � ||�}t
|�}| � |� t
|�}|||< |d kr|}q||fS )N�:) �typer � ENDMARKER�NEWLINEr �expectrA �OP� parse_rhs�make_dfar# � simplify_dfa) r r r r1 �a�zr3 ZoldlenZnewlenr r r
r � s"
zParserGenerator.parsec s� � fdd�}� fdd�� t ||�|�g}|D ]�}i }|jD ].}|jD ]"\}} |d k rD� | |�|i �� qDq:t|�� �D ]@\}}
|D ]} | j|
kr� q�q�t |
|�} |�| � |�| |� qvq,|S )Nc s i }� | |� |S r r )r4 �base��
addclosurer r
�closure� s
z)ParserGenerator.make_dfa.<locals>.closurec s: | |kr d S d|| < | j D ]\}}|d kr� ||� qd S r8 �r& )r4 rR r5 r6 rS r r
rT � s z,ParserGenerator.make_dfa.<locals>.addclosure)�DFAState�nfasetr& �
setdefaultr% r' r( �addarc) r r. �finishrU r, r4 r& Znfastater5 r6 rX �str rS r
rN � s"
zParserGenerator.make_dfac
C s� t d|� |g}t|�D ]|\}}t d|||kr2dp4d� |jD ]T\}}||krZ|�|�} nt|�} |�|� |d kr�t d| � q>t d|| f � q>qd S )NzDump of NFA for� State�(final)� z -> %d� %s -> %d)�print� enumerater& r* r# r( )
r r1 r. r[ Ztodor2 r4 r5 r6 �jr r r
�dump_nfa� s
zParserGenerator.dump_nfac C sd t d|� t|�D ]L\}}t d||jr*dp,d� t|j�� �D ] \}}t d||�|�f � q>qd S )NzDump of DFA forr] r^ r_ r` )ra rb r+ r% r& r' r* )r r1 r3 r2 r4 r5 r6 r r r
�dump_dfa� s
zParserGenerator.dump_dfac C sp d}|rld}t |�D ]T\}}t|d t|��D ]8}|| }||kr.||= |D ]}|�||� qLd} qq.qqd S )NTFr )rb �ranger# �
unifystate)r r3 Zchangesr2 Zstate_irc Zstate_jr4 r r r
rO � s z ParserGenerator.simplify_dfac C s~ | � � \}}| jdkr ||fS t� }t� }|�|� |�|� | jdkrr| �� | � � \}}|�|� |�|� q>||fS d S )N�|)� parse_altrB �NFAStaterZ r )r rP rQ ZaaZzzr r r
rM � s
zParserGenerator.parse_rhsc C sL | � � \}}| jdks(| jtjtjfkrD| � � \}}|�|� |}q ||fS )N)�(�[)�
parse_itemrB rH r rA �STRINGrZ )r rP �br/ �dr r r
ri
s
�
zParserGenerator.parse_altc C s� | j dkr>| �� | �� \}}| �tjd� |�|� ||fS | �� \}}| j }|dkr`||fS | �� |�|� |dkr�||fS ||fS d S )Nrl �])�+�*rr )rB r rM rK r rL rZ �
parse_atom)r rP rQ rB r r r
rm s
zParserGenerator.parse_itemc C s� | j dkr4| �� | �� \}}| �tjd� ||fS | jtjtjfkrpt � }t � }|�
|| j � | �� ||fS | � d| j| j � d S )Nrk �)z+expected (...) or NAME or STRING, got %s/%s) rB r rM rK r rL rH rA rn rj rZ � raise_error)r rP rQ r r r
rt ( s
�zParserGenerator.parse_atomc C sD | j |ks |d k r2| j|kr2| �d||| j | j� | j}| �� |S )Nzexpected %s/%s, got %s/%s)rH rB rv r )r rH rB r r r
rK 9 s �zParserGenerator.expectc C sF t | j�}|d tjtjfkr*t | j�}q
|\| _| _| _| _| _ d S r: )
r6 r r �COMMENT�NLrH rB Zbegin�end�line)r �tupr r r
r A s
zParserGenerator.gettokenc
G s^ |r8z || }W n& d� |gttt|�� �}Y nX t|| j| jd | jd | jf��d S )N� r r )�joinr �map�str� SyntaxErrorr ry rz )r �msg�argsr r r
rv H s �zParserGenerator.raise_error)N)N)r r r r r7 r- r) r rC r rN rd re rO rM ri rm rt rK r rv r r r r
r
s$
. $
r c @ s e Zd Zdd� Zddd�ZdS )rj c C s
g | _ d S r rV )r r r r
r S s zNFAState.__init__Nc C s | j �||f� d S r )r& r( �r r6 r5 r r r
rZ V s zNFAState.addarc)N)r r r r rZ r r r r
rj Q s rj c @ s0 e Zd Zdd� Zdd� Zdd� Zdd� Zd Zd S )
rW c C s || _ ||k| _i | _d S r )rX r+ r& )r rX �finalr r r
r ] s
zDFAState.__init__c C s || j |< d S r rV r� r r r
rZ e s zDFAState.addarcc C s* | j �� D ]\}}||kr
|| j |< q
d S r )r& r' )r �old�newr5 r6 r r r
rg k s zDFAState.unifystatec C sV | j |j krdS t| j�t|j�kr(dS | j�� D ] \}}||j�|�k r2 dS q2dS )NFT)r+ r# r& r' �get)r �otherr5 r6 r r r
�__eq__p s zDFAState.__eq__N)r r r r rZ rg r� �__hash__r r r r
rW [ s
rW � Grammar.txtc C s t | �}|�� S r )r r7 )r �pr r r
�generate_grammar� s r� N)r� ) r_ r r r ZGrammarr �objectr rj rW r� r r r r
�<module> s I
% __pycache__/conv.cpython-38.opt-1.pyc 0000644 00000013774 15052147326 0013331 0 ustar 00 U
e5d�% � @ s2 d Z ddlZddlmZmZ G dd� dej�ZdS )a� Convert graminit.[ch] spit out by pgen to Python code.
Pgen is the Python parser generator. It is useful to quickly create a
parser from a grammar file in Python's grammar notation. But I don't
want my parsers to be written in C (yet), so I'm translating the
parsing tables to Python data structures and writing a Python parse
engine.
Note that the token numbers are constants determined by the standard
Python tokenizer. The standard token module defines these numbers and
their names (the names are not used much). The token numbers are
hardcoded into the Python tokenizer and into pgen. A Python
implementation of the Python tokenizer is also available, in the
standard tokenize module.
On the other hand, symbol numbers (representing the grammar's
non-terminals) are assigned by pgen based on the actual grammar
input.
Note: this module is pretty much obsolete; the pgen module generates
equivalent grammar tables directly from the Grammar.txt input file
without having to invoke the Python pgen C program.
� N)�grammar�tokenc @ s0 e Zd ZdZdd� Zdd� Zdd� Zdd � Zd
S ) � Convertera2 Grammar subclass that reads classic pgen output files.
The run() method reads the tables as produced by the pgen parser
generator, typically contained in two C files, graminit.h and
graminit.c. The other methods are for internal use only.
See the base class for more documentation.
c C s | � |� | �|� | �� dS )z<Load the grammar tables from the text files written by pgen.N)�parse_graminit_h�parse_graminit_c�
finish_off)�selfZ
graminit_hZ
graminit_c� r �*/usr/lib64/python3.8/lib2to3/pgen2/conv.py�run/ s
z
Converter.runc
C s� z t |�}W n8 tk
rD } ztd||f � W Y �dS d}~X Y nX i | _i | _d}|D ]d}|d7 }t�d|�}|s�|�� r�td|||�� f � qZ|�� \}}t |�}|| j|< || j|< qZdS ) z�Parse the .h file written by pgen. (Internal)
This file is a sequence of #define statements defining the
nonterminals of the grammar as numbers. We build two tables
mapping the numbers to names and back.
�Can't open %s: %sFNr � z^#define\s+(\w+)\s+(\d+)$z%s(%s): can't parse %sT)
�open�OSError�printZ
symbol2numberZ
number2symbol�re�match�strip�groups�int) r �filename�f�err�lineno�line�mo�symbol�numberr r r
r 5 s( �
zConverter.parse_graminit_hc !
C s� z t |�}W n8 tk
rD } ztd||f � W Y �dS d}~X Y nX d}|d t|� }}|d t|� }}|d t|� }}i }g }|�d��r�|�d��rJt�d|�}ttt |�
� ��\} }
} g } t | �D ]F}
|d t|� }}t�d|�}ttt |�
� ��\}}| � ||f� q�|d t|� }}| || |
f< |d t|� }}q�t�d |�}ttt |�
� ��\}}g }t |�D ]R}
|d t|� }}t�d
|�}ttt |�
� ��\} } }
|| |
f } |� | � �qx|� |� |d t|� }}|d t|� }}q�|| _
i }t�d |�}t |�d��}t |�D ]�}|d t|� }}t�d |�}|�d
�}ttt |�dddd���\}}}}|| }|d t|� }}t�d|�}i }t|�d��}t|�D ]@\}} t| �} t d�D ]$}| d|> @ �r�d||d | < �qΐq�||f||< �q(|d t|� }}|| _g } |d t|� }}t�d|�}t |�d��} t | �D ]^}|d t|� }}t�d|�}|�
� \}}t |�}|dk�r�d}nt|�}| � ||f� �qX|d t|� }}| | _|d t|� }}|d t|� }}t�d|�}t |�d��}|d t|� }}|d t|� }}t�d|�}t |�d��} |d t|� }}t�d|�}t |�d��} | | _|d t|� }}z|d t|� }}W n tk
�r� Y nX dS )a� Parse the .c file written by pgen. (Internal)
The file looks as follows. The first two lines are always this:
#include "pgenheaders.h"
#include "grammar.h"
After that come four blocks:
1) one or more state definitions
2) a table defining dfas
3) a table defining labels
4) a struct defining the grammar
A state definition has the following form:
- one or more arc arrays, each of the form:
static arc arcs_<n>_<m>[<k>] = {
{<i>, <j>},
...
};
- followed by a state array, of the form:
static state states_<s>[<t>] = {
{<k>, arcs_<n>_<m>},
...
};
r FNr r
z static arc z)static arc arcs_(\d+)_(\d+)\[(\d+)\] = {$z\s+{(\d+), (\d+)},$z'static state states_(\d+)\[(\d+)\] = {$z \s+{(\d+), arcs_(\d+)_(\d+)},$z static dfa dfas\[(\d+)\] = {$z0\s+{(\d+), "(\w+)", (\d+), (\d+), states_(\d+),$� � � � z\s+("(?:\\\d\d\d)*")},$� z!static label labels\[(\d+)\] = {$z\s+{(\d+), (0|"\w+")},$�0z
\s+(\d+),$z\s+{(\d+), labels},$z \s+(\d+)$)r r r �next�
startswithr r �list�mapr r �range�append�states�group�eval� enumerate�ord�dfas�labels�start�
StopIteration)!r r r r r r Zallarcsr* r �n�m�kZarcs�_�i�j�s�t�stater/ Zndfasr r �x�y�z�firstZ rawbitset�cZbyter0 Znlabelsr1 r r r
r T s� �
�
"
zConverter.parse_graminit_cc C sX i | _ i | _t| j�D ]<\}\}}|tjkr@|dk r@|| j |< q|dkr|| j|< qdS )z1Create additional useful structures. (Internal).N)�keywords�tokensr- r0 r �NAME)r Zilabel�type�valuer r r
r � s zConverter.finish_offN)�__name__�
__module__� __qualname__�__doc__r r r r r r r r
r $ s
&r )rI r Zpgen2r r ZGrammarr r r r r
�<module> s __pycache__/parse.cpython-38.opt-2.pyc 0000644 00000006256 15052147326 0013474 0 ustar 00 U
e5d� � @ s0 d dl mZ G dd� de�ZG dd� de�ZdS )� )�tokenc @ s e Zd Zdd� Zdd� ZdS )�
ParseErrorc C s4 t �| d||||f � || _|| _|| _|| _d S )Nz!%s: type=%r, value=%r, context=%r)� Exception�__init__�msg�type�value�context)�selfr r r r � r �+/usr/lib64/python3.8/lib2to3/pgen2/parse.pyr s
�zParseError.__init__c C s t | �| j| j | j| jffS �N)r r r r )r
r r r �
__reduce__ s zParseError.__reduce__N)�__name__�
__module__� __qualname__r r r r r r r s r c @ sH e Zd Zddd�Zddd�Zdd� Zdd � Zd
d � Zd d
� Zdd� Z dS )�ParserNc C s || _ |pdd� | _d S )Nc S s |S r
r )�grammar�noder r r �<lambda>Z � z!Parser.__init__.<locals>.<lambda>)r �convert)r
r r r r r r < s zParser.__init__c C sH |d kr| j j}|d d g f}| j j| d|f}|g| _d | _t� | _d S )N� )r �start�dfas�stack�rootnode�set�
used_names)r
r �newnodeZ
stackentryr r r �setup\ s
z Parser.setupc C s0 | � |||�}| jd \}}}|\}} || }
|
D ]�\} } | jj| \}
}|| kr�| �||| |� | }|| d|fgkr�| �� | js� dS | jd \}}}|\}} qf dS |
dkr2| jj|
}|\}}||kr2| �|
| jj|
| |� qq2d|f|
k�r | �� | j�s*td|||��qtd|||��qd S )N���r TF� ztoo much inputz bad input) �classifyr r �labels�shift�popr �pushr )r
r r r �ilabel�dfa�stater Zstates�firstZarcs�i�newstate�t�vZitsdfaZ itsstatesZitsfirstr r r �addtokent s>
�zParser.addtokenc C sX |t jkr0| j�|� | jj�|�}|d k r0|S | jj�|�}|d krTtd|||��|S )Nz bad token) r �NAMEr �addr �keywords�get�tokensr )r
r r r r( r r r r# � s
zParser.classifyc C sT | j d \}}}|||d f}| �| j|�}|d k r@|d �|� |||f| j d< d S �Nr! )r r r �append) r
r r r- r r) r* r r r r r r% � s z Parser.shiftc C sB | j d \}}}|d |g f}|||f| j d< | j �|d|f� d S )Nr! r )r r7 ) r
r Znewdfar- r r) r* r r r r r r' � s z Parser.pushc C s` | j �� \}}}| �| j|�}|d k r\| j rL| j d \}}}|d �|� n|| _| j| j_d S r6 )r r& r r r7 r r )r
ZpopdfaZpopstateZpopnoder r) r* r r r r r& � s z
Parser.pop)N)N)
r r r r r r0 r# r% r' r& r r r r r s
0 r N)� r r r �objectr r r r r �<module> s __pycache__/grammar.cpython-38.opt-1.pyc 0000644 00000013043 15052147326 0013777 0 ustar 00 U
e5d� � @ s` d Z ddlZddlmZ G dd� de�ZdZi Ze�� D ]"Z e r8e �
� \Z Z e
ee �ee < q8dS )a� This module defines the data structures used to represent a grammar.
These are a bit arcane because they are derived from the data
structures used by Python's 'pgen' parser generator.
There's also a table here mapping operators to their names in the
token module; the Python tokenize module reports all operators as the
fallback token code OP, but the parser needs the actual token code.
� N� )�tokenc @ s@ e Zd ZdZdd� Zdd� Zdd� Zdd � Zd
d � Zd d
� Z dS )�Grammara� Pgen parsing tables conversion class.
Once initialized, this class supplies the grammar tables for the
parsing engine implemented by parse.py. The parsing engine
accesses the instance variables directly. The class here does not
provide initialization of the tables; several subclasses exist to
do this (see the conv and pgen modules).
The load() method reads the tables from a pickle file, which is
much faster than the other ways offered by subclasses. The pickle
file is written by calling dump() (after loading the grammar
tables using a subclass). The report() method prints a readable
representation of the tables to stdout, for debugging.
The instance variables are as follows:
symbol2number -- a dict mapping symbol names to numbers. Symbol
numbers are always 256 or higher, to distinguish
them from token numbers, which are between 0 and
255 (inclusive).
number2symbol -- a dict mapping numbers to symbol names;
these two are each other's inverse.
states -- a list of DFAs, where each DFA is a list of
states, each state is a list of arcs, and each
arc is a (i, j) pair where i is a label and j is
a state number. The DFA number is the index into
this list. (This name is slightly confusing.)
Final states are represented by a special arc of
the form (0, j) where j is its own state number.
dfas -- a dict mapping symbol numbers to (DFA, first)
pairs, where DFA is an item from the states list
above, and first is a set of tokens that can
begin this grammar rule (represented by a dict
whose values are always 1).
labels -- a list of (x, y) pairs where x is either a token
number or a symbol number, and y is either None
or a string; the strings are keywords. The label
number is the index in this list; label numbers
are used to mark state transitions (arcs) in the
DFAs.
start -- the number of the grammar's start symbol.
keywords -- a dict mapping keyword strings to arc labels.
tokens -- a dict mapping token numbers to arc labels.
c C s< i | _ i | _g | _i | _dg| _i | _i | _i | _d| _d S )N)r ZEMPTY� ) �
symbol2number�
number2symbol�states�dfas�labels�keywords�tokens� symbol2label�start)�self� r �-/usr/lib64/python3.8/lib2to3/pgen2/grammar.py�__init__L s zGrammar.__init__c C s, t |d��}t�| j|tj� W 5 Q R X dS )z)Dump the grammar tables to a pickle file.�wbN)�open�pickle�dump�__dict__ZHIGHEST_PROTOCOL)r �filename�fr r r r W s z Grammar.dumpc C s0 t |d��}t�|�}W 5 Q R X | j�|� dS )z+Load the grammar tables from a pickle file.�rbN)r r �loadr �update)r r r �dr r r r \ s z Grammar.loadc C s | j �t�|�� dS )z3Load the grammar tables from a pickle bytes object.N)r r r �loads)r Zpklr r r r b s z
Grammar.loadsc C sT | � � }dD ]}t||t| |��� � q | jdd� |_| jdd� |_| j|_|S )z#
Copy the grammar.
)r r r r r r
N)� __class__�setattr�getattr�copyr
r r )r �newZ dict_attrr r r r" f s z Grammar.copyc C sv ddl m } td� || j� td� || j� td� || j� td� || j� td� || j� td| j� d S )
z:Dump the grammar tables to standard output, for debugging.r )�pprintZs2nZn2sr r r
r N)r$ �printr r r r r
r )r r$ r r r �reports s
zGrammar.reportN)
�__name__�
__module__� __qualname__�__doc__r r r r r" r& r r r r r s 5
r a
( LPAR
) RPAR
[ LSQB
] RSQB
: COLON
, COMMA
; SEMI
+ PLUS
- MINUS
* STAR
/ SLASH
| VBAR
& AMPER
< LESS
> GREATER
= EQUAL
. DOT
% PERCENT
` BACKQUOTE
{ LBRACE
} RBRACE
@ AT
@= ATEQUAL
== EQEQUAL
!= NOTEQUAL
<> NOTEQUAL
<= LESSEQUAL
>= GREATEREQUAL
~ TILDE
^ CIRCUMFLEX
<< LEFTSHIFT
>> RIGHTSHIFT
** DOUBLESTAR
+= PLUSEQUAL
-= MINEQUAL
*= STAREQUAL
/= SLASHEQUAL
%= PERCENTEQUAL
&= AMPEREQUAL
|= VBAREQUAL
^= CIRCUMFLEXEQUAL
<<= LEFTSHIFTEQUAL
>>= RIGHTSHIFTEQUAL
**= DOUBLESTAREQUAL
// DOUBLESLASH
//= DOUBLESLASHEQUAL
-> RARROW
:= COLONEQUAL
)r* r � r �objectr Z opmap_rawZopmap�
splitlines�line�split�op�namer! r r r r �<module> s o3 __pycache__/grammar.cpython-38.opt-2.pyc 0000644 00000004724 15052147326 0014006 0 ustar 00 U
e5d� � @ s\ d dl Z ddlmZ G dd� de�ZdZi Ze�� D ]"Zer4e� � \Z
Z e ee �ee
< q4dS )� N� )�tokenc @ s< e Zd Zdd� Zdd� Zdd� Zdd� Zd d
� Zd d � Zd
S )�Grammarc C s< i | _ i | _g | _i | _dg| _i | _i | _i | _d| _d S )N)r ZEMPTY� ) �
symbol2number�
number2symbol�states�dfas�labels�keywords�tokens� symbol2label�start)�self� r �-/usr/lib64/python3.8/lib2to3/pgen2/grammar.py�__init__L s zGrammar.__init__c C s, t |d��}t�| j|tj� W 5 Q R X d S )N�wb)�open�pickle�dump�__dict__ZHIGHEST_PROTOCOL)r �filename�fr r r r W s z Grammar.dumpc C s0 t |d��}t�|�}W 5 Q R X | j�|� d S )N�rb)r r �loadr �update)r r r �dr r r r \ s z Grammar.loadc C s | j �t�|�� d S )N)r r r �loads)r Zpklr r r r b s z
Grammar.loadsc C sT | � � }dD ]}t||t| |��� � q | jd d � |_| jd d � |_| j|_|S )N)r r r r r r
)� __class__�setattr�getattr�copyr
r r )r �newZ dict_attrr r r r" f s z Grammar.copyc C sv ddl m } td� || j� td� || j� td� || j� td� || j� td� || j� td| j� d S ) Nr )�pprintZs2nZn2sr r r
r )r$ �printr r r r r
r )r r$ r r r �reports s
zGrammar.reportN) �__name__�
__module__� __qualname__r r r r r" r& r r r r r s 6
r a
( LPAR
) RPAR
[ LSQB
] RSQB
: COLON
, COMMA
; SEMI
+ PLUS
- MINUS
* STAR
/ SLASH
| VBAR
& AMPER
< LESS
> GREATER
= EQUAL
. DOT
% PERCENT
` BACKQUOTE
{ LBRACE
} RBRACE
@ AT
@= ATEQUAL
== EQEQUAL
!= NOTEQUAL
<> NOTEQUAL
<= LESSEQUAL
>= GREATEREQUAL
~ TILDE
^ CIRCUMFLEX
<< LEFTSHIFT
>> RIGHTSHIFT
** DOUBLESTAR
+= PLUSEQUAL
-= MINEQUAL
*= STAREQUAL
/= SLASHEQUAL
%= PERCENTEQUAL
&= AMPEREQUAL
|= VBAREQUAL
^= CIRCUMFLEXEQUAL
<<= LEFTSHIFTEQUAL
>>= RIGHTSHIFTEQUAL
**= DOUBLESTAREQUAL
// DOUBLESLASH
//= DOUBLESLASHEQUAL
-> RARROW
:= COLONEQUAL
)
r � r �objectr Z opmap_rawZopmap�
splitlines�line�split�op�namer! r r r r �<module> s o3 __pycache__/__init__.cpython-38.pyc 0000644 00000000247 15052147326 0013153 0 ustar 00 U
e5d� � @ s d Z dS )zThe pgen2 package.N)�__doc__� r r �./usr/lib64/python3.8/lib2to3/pgen2/__init__.py�<module> �