Skip to content

Commit f1c2c69

Browse files
ncwncw
committed
compiler: symbol table framework, plus test machinery
1 parent 611c8dc commit f1c2c69

File tree

6 files changed

+535
-9
lines changed

6 files changed

+535
-9
lines changed

‎compile/compile_data_test.go

Lines changed: 41 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1151,7 +1151,7 @@ var compileTestData = []struct {
11511151
Name: "<module>",
11521152
Firstlineno: 1,
11531153
Lnotab: "",
1154-
}, " 1 0 LOAD_CONST 0 (<code object <lambda> at 0x7fe734191db0, file \"<string>\", line 1>)\n 3 LOAD_CONST 1 ('<lambda>')\n 6 MAKE_FUNCTION 0\n 9 RETURN_VALUE\n", nil, ""},
1154+
}, " 1 0 LOAD_CONST 0 (<code object <lambda> at 0x7fd7bccdcdb0, file \"<string>\", line 1>)\n 3 LOAD_CONST 1 ('<lambda>')\n 6 MAKE_FUNCTION 0\n 9 RETURN_VALUE\n", nil, ""},
11551155
{"pass", "exec", &py.Code{
11561156
Argcount: 0,
11571157
Kwonlyargcount: 0,
@@ -1951,7 +1951,7 @@ var compileTestData = []struct {
19511951
Name: "<module>",
19521952
Firstlineno: 1,
19531953
Lnotab: "",
1954-
}, " 1 0 LOAD_CONST 0 (<code object fn at 0x7fe7341918a0, file \"<string>\", line 1>)\n 3 LOAD_CONST 1 ('fn')\n 6 MAKE_FUNCTION 0\n 9 STORE_NAME 0 (fn)\n 12 LOAD_CONST 2 (None)\n 15 RETURN_VALUE\n", nil, ""},
1954+
}, " 1 0 LOAD_CONST 0 (<code object fn at 0x7fd7bccdc8a0, file \"<string>\", line 1>)\n 3 LOAD_CONST 1 ('fn')\n 6 MAKE_FUNCTION 0\n 9 STORE_NAME 0 (fn)\n 12 LOAD_CONST 2 (None)\n 15 RETURN_VALUE\n", nil, ""},
19551955
{"def fn(a): pass", "exec", &py.Code{
19561956
Argcount: 0,
19571957
Kwonlyargcount: 0,
@@ -1984,7 +1984,7 @@ var compileTestData = []struct {
19841984
Name: "<module>",
19851985
Firstlineno: 1,
19861986
Lnotab: "",
1987-
}, " 1 0 LOAD_CONST 0 (<code object fn at 0x7fe73567c6f0, file \"<string>\", line 1>)\n 3 LOAD_CONST 1 ('fn')\n 6 MAKE_FUNCTION 0\n 9 STORE_NAME 0 (fn)\n 12 LOAD_CONST 2 (None)\n 15 RETURN_VALUE\n", nil, ""},
1987+
}, " 1 0 LOAD_CONST 0 (<code object fn at 0x7fd7be1c76f0, file \"<string>\", line 1>)\n 3 LOAD_CONST 1 ('fn')\n 6 MAKE_FUNCTION 0\n 9 STORE_NAME 0 (fn)\n 12 LOAD_CONST 2 (None)\n 15 RETURN_VALUE\n", nil, ""},
19881988
{"def fn(a,b,c): pass", "exec", &py.Code{
19891989
Argcount: 0,
19901990
Kwonlyargcount: 0,
@@ -2017,7 +2017,7 @@ var compileTestData = []struct {
20172017
Name: "<module>",
20182018
Firstlineno: 1,
20192019
Lnotab: "",
2020-
}, " 1 0 LOAD_CONST 0 (<code object fn at 0x7fe734191db0, file \"<string>\", line 1>)\n 3 LOAD_CONST 1 ('fn')\n 6 MAKE_FUNCTION 0\n 9 STORE_NAME 0 (fn)\n 12 LOAD_CONST 2 (None)\n 15 RETURN_VALUE\n", nil, ""},
2020+
}, " 1 0 LOAD_CONST 0 (<code object fn at 0x7fd7bccdcdb0, file \"<string>\", line 1>)\n 3 LOAD_CONST 1 ('fn')\n 6 MAKE_FUNCTION 0\n 9 STORE_NAME 0 (fn)\n 12 LOAD_CONST 2 (None)\n 15 RETURN_VALUE\n", nil, ""},
20212021
{"def fn(a,b=1,c=2): pass", "exec", &py.Code{
20222022
Argcount: 0,
20232023
Kwonlyargcount: 0,
@@ -2050,7 +2050,7 @@ var compileTestData = []struct {
20502050
Name: "<module>",
20512051
Firstlineno: 1,
20522052
Lnotab: "",
2053-
}, " 1 0 LOAD_CONST 0 (1)\n 3 LOAD_CONST 1 (2)\n 6 LOAD_CONST 2 (<code object fn at 0x7fe7341b68a0, file \"<string>\", line 1>)\n 9 LOAD_CONST 3 ('fn')\n 12 MAKE_FUNCTION 2\n 15 STORE_NAME 0 (fn)\n 18 LOAD_CONST 4 (None)\n 21 RETURN_VALUE\n", nil, ""},
2053+
}, " 1 0 LOAD_CONST 0 (1)\n 3 LOAD_CONST 1 (2)\n 6 LOAD_CONST 2 (<code object fn at 0x7fd7bcd028a0, file \"<string>\", line 1>)\n 9 LOAD_CONST 3 ('fn')\n 12 MAKE_FUNCTION 2\n 15 STORE_NAME 0 (fn)\n 18 LOAD_CONST 4 (None)\n 21 RETURN_VALUE\n", nil, ""},
20542054
{"def fn(a,*arg,b=1,c=2): pass", "exec", &py.Code{
20552055
Argcount: 0,
20562056
Kwonlyargcount: 0,
@@ -2083,7 +2083,7 @@ var compileTestData = []struct {
20832083
Name: "<module>",
20842084
Firstlineno: 1,
20852085
Lnotab: "",
2086-
}, " 1 0 LOAD_CONST 0 ('b')\n 3 LOAD_CONST 1 (1)\n 6 LOAD_CONST 2 ('c')\n 9 LOAD_CONST 3 (2)\n 12 LOAD_CONST 4 (<code object fn at 0x7fe7341918a0, file \"<string>\", line 1>)\n 15 LOAD_CONST 5 ('fn')\n 18 MAKE_FUNCTION 512\n 21 STORE_NAME 0 (fn)\n 24 LOAD_CONST 6 (None)\n 27 RETURN_VALUE\n", nil, ""},
2086+
}, " 1 0 LOAD_CONST 0 ('b')\n 3 LOAD_CONST 1 (1)\n 6 LOAD_CONST 2 ('c')\n 9 LOAD_CONST 3 (2)\n 12 LOAD_CONST 4 (<code object fn at 0x7fd7bccdc8a0, file \"<string>\", line 1>)\n 15 LOAD_CONST 5 ('fn')\n 18 MAKE_FUNCTION 512\n 21 STORE_NAME 0 (fn)\n 24 LOAD_CONST 6 (None)\n 27 RETURN_VALUE\n", nil, ""},
20872087
{"def fn(a,*arg,b=1,c=2,**kwargs): pass", "exec", &py.Code{
20882088
Argcount: 0,
20892089
Kwonlyargcount: 0,
@@ -2116,7 +2116,7 @@ var compileTestData = []struct {
21162116
Name: "<module>",
21172117
Firstlineno: 1,
21182118
Lnotab: "",
2119-
}, " 1 0 LOAD_CONST 0 ('b')\n 3 LOAD_CONST 1 (1)\n 6 LOAD_CONST 2 ('c')\n 9 LOAD_CONST 3 (2)\n 12 LOAD_CONST 4 (<code object fn at 0x7fe73567c6f0, file \"<string>\", line 1>)\n 15 LOAD_CONST 5 ('fn')\n 18 MAKE_FUNCTION 512\n 21 STORE_NAME 0 (fn)\n 24 LOAD_CONST 6 (None)\n 27 RETURN_VALUE\n", nil, ""},
2119+
}, " 1 0 LOAD_CONST 0 ('b')\n 3 LOAD_CONST 1 (1)\n 6 LOAD_CONST 2 ('c')\n 9 LOAD_CONST 3 (2)\n 12 LOAD_CONST 4 (<code object fn at 0x7fd7be1c76f0, file \"<string>\", line 1>)\n 15 LOAD_CONST 5 ('fn')\n 18 MAKE_FUNCTION 512\n 21 STORE_NAME 0 (fn)\n 24 LOAD_CONST 6 (None)\n 27 RETURN_VALUE\n", nil, ""},
21202120
{"def fn(a:\"a\",*arg:\"arg\",b:\"b\"=1,c:\"c\"=2,**kwargs:\"kw\") -> \"ret\": pass", "exec", &py.Code{
21212121
Argcount: 0,
21222122
Kwonlyargcount: 0,
@@ -2149,5 +2149,38 @@ var compileTestData = []struct {
21492149
Name: "<module>",
21502150
Firstlineno: 1,
21512151
Lnotab: "",
2152-
}, " 1 0 LOAD_CONST 0 ('b')\n 3 LOAD_CONST 1 (1)\n 6 LOAD_CONST 2 ('c')\n 9 LOAD_CONST 3 (2)\n 12 LOAD_CONST 4 ('a')\n 15 LOAD_CONST 5 ('arg')\n 18 LOAD_CONST 0 ('b')\n 21 LOAD_CONST 2 ('c')\n 24 LOAD_CONST 6 ('kw')\n 27 LOAD_CONST 7 ('ret')\n 30 LOAD_CONST 8 (('a', 'arg', 'b', 'c', 'kwargs', 'return'))\n 33 LOAD_CONST 9 (<code object fn at 0x7fe734191db0, file \"<string>\", line 1>)\n 36 LOAD_CONST 10 ('fn')\n 39 EXTENDED_ARG 7\n 42 MAKE_FUNCTION 459264\n 45 STORE_NAME 0 (fn)\n 48 LOAD_CONST 11 (None)\n 51 RETURN_VALUE\n", nil, ""},
2152+
}, " 1 0 LOAD_CONST 0 ('b')\n 3 LOAD_CONST 1 (1)\n 6 LOAD_CONST 2 ('c')\n 9 LOAD_CONST 3 (2)\n 12 LOAD_CONST 4 ('a')\n 15 LOAD_CONST 5 ('arg')\n 18 LOAD_CONST 0 ('b')\n 21 LOAD_CONST 2 ('c')\n 24 LOAD_CONST 6 ('kw')\n 27 LOAD_CONST 7 ('ret')\n 30 LOAD_CONST 8 (('a', 'arg', 'b', 'c', 'kwargs', 'return'))\n 33 LOAD_CONST 9 (<code object fn at 0x7fd7bccdcdb0, file \"<string>\", line 1>)\n 36 LOAD_CONST 10 ('fn')\n 39 EXTENDED_ARG 7\n 42 MAKE_FUNCTION 459264\n 45 STORE_NAME 0 (fn)\n 48 LOAD_CONST 11 (None)\n 51 RETURN_VALUE\n", nil, ""},
2153+
{"def fn(): a+b", "exec", &py.Code{
2154+
Argcount: 0,
2155+
Kwonlyargcount: 0,
2156+
Nlocals: 0,
2157+
Stacksize: 2,
2158+
Flags: 64,
2159+
Code: "\x64\x00\x00\x64\x01\x00\x84\x00\x00\x5a\x00\x00\x64\x02\x00\x53",
2160+
Consts: []py.Object{&py.Code{
2161+
Argcount: 0,
2162+
Kwonlyargcount: 0,
2163+
Nlocals: 0,
2164+
Stacksize: 2,
2165+
Flags: 67,
2166+
Code: "\x74\x00\x00\x74\x01\x00\x17\x01\x64\x00\x00\x53",
2167+
Consts: []py.Object{py.None},
2168+
Names: []string{"a", "b"},
2169+
Varnames: []string{},
2170+
Freevars: []string{},
2171+
Cellvars: []string{},
2172+
Filename: "<string>",
2173+
Name: "fn",
2174+
Firstlineno: 1,
2175+
Lnotab: "",
2176+
}, py.String("fn"), py.None},
2177+
Names: []string{"fn"},
2178+
Varnames: []string{},
2179+
Freevars: []string{},
2180+
Cellvars: []string{},
2181+
Filename: "<string>",
2182+
Name: "<module>",
2183+
Firstlineno: 1,
2184+
Lnotab: "",
2185+
}, " 1 0 LOAD_CONST 0 (<code object fn at 0x7fd7bcd028a0, file \"<string>\", line 1>)\n 3 LOAD_CONST 1 ('fn')\n 6 MAKE_FUNCTION 0\n 9 STORE_NAME 0 (fn)\n 12 LOAD_CONST 2 (None)\n 15 RETURN_VALUE\n", nil, ""},
21532186
}

‎compile/make_compile_test.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -164,7 +164,7 @@
164164
('''def fn(a,*arg,b=1,c=2): pass''', "exec"),
165165
('''def fn(a,*arg,b=1,c=2,**kwargs): pass''', "exec"),
166166
('''def fn(a:"a",*arg:"arg",b:"b"=1,c:"c"=2,**kwargs:"kw") -> "ret": pass''', "exec"),
167-
#('''def fn(): a+b''', "exec"),
167+
('''def fn(): a+b''', "exec"),
168168

169169
]
170170

‎compile/make_symtable_test.py

Lines changed: 158 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,158 @@
1+
#!/usr/bin/env python3.4
2+
"""
3+
Write symtable_data_test.go
4+
"""
5+
6+
import sys
7+
import ast
8+
import subprocess
9+
import dis
10+
from symtable import symtable
11+
12+
# FIXME test errors too
13+
14+
inp = [
15+
('''1''', "eval"),
16+
#('''a*b*c''', "eval"),
17+
#('''def fn(): pass''', "exec"),
18+
#('''def fn(a,b):\n e=1\n return a*b*c*d*e''', "exec"),
19+
#('''def fn(a,b):\n def nested(c,d):\n return a*b*c*d*e''', "exec"),
20+
# ('''def fn(a,b,c): pass''', "exec"),
21+
# ('''def fn(a,b=1,c=2): pass''', "exec"),
22+
# ('''def fn(a,*arg,b=1,c=2): pass''', "exec"),
23+
# ('''def fn(a,*arg,b=1,c=2,**kwargs): pass''', "exec"),
24+
# ('''def fn(a:"a",*arg:"arg",b:"b"=1,c:"c"=2,**kwargs:"kw") -> "ret": pass''', "exec"),
25+
# ('''def fn(): a+b''', "exec"),
26+
]
27+
28+
def dump_bool(b):
29+
return ("true" if b else "false")
30+
31+
def dump_strings(ss):
32+
return "[]string{"+",".join([ '"%s"' % s for s in ss ])+"}"
33+
34+
# Scope numbers to names (from symtable.h)
35+
SCOPES = {
36+
1: "scopeLocal",
37+
2: "scopeGlobalExplicit",
38+
3: "scopeGlobalImplicit",
39+
4: "scopeFree",
40+
5: "scopeCell",
41+
}
42+
43+
#def-use flags to names (from symtable.h)
44+
DEF_FLAGS = (
45+
("defGlobal", 1), # global stmt
46+
("defLocal", 2), # assignment in code block
47+
("defParam", 2<<1), # formal parameter
48+
("defNonlocal", 2<<2), # nonlocal stmt
49+
("defUse", 2<<3), # name is used
50+
("defFree", 2<<4), # name used but not defined in nested block
51+
("defFreeClass", 2<<5),# free variable from class's method
52+
("defImport", 2<<6), # assignment occurred via import
53+
)
54+
55+
BLOCK_TYPES = {
56+
"function": "FunctionBlock",
57+
"class": "ClassBlock",
58+
"module": "ModuleBlock",
59+
}
60+
61+
def dump_symtable(st):
62+
"""Dump the symtable"""
63+
out = "&SymTable{\n"
64+
out += 'Type:%s,\n' % BLOCK_TYPES[st.get_type()] # Return the type of the symbol table. Possible values are 'class', 'module', and 'function'.
65+
out += 'Name:"%s",\n' % st.get_name() # Return the table’s name. This is the name of the class if the table is for a class, the name of the function if the table is for a function, or 'top' if the table is global (get_type() returns 'module').
66+
67+
out += 'Lineno:%s,\n' % st.get_lineno() # Return the number of the first line in the block this table represents.
68+
out += 'Optimized:%s,\n' % dump_bool(st.is_optimized()) # Return True if the locals in this table can be optimized.
69+
out += 'Nested:%s,\n' % dump_bool(st.is_nested()) # Return True if the block is a nested class or function.
70+
out += 'Exec:%s,\n' % dump_bool(st.has_exec()) # Return True if the block uses exec.
71+
out += 'ImportStar:%s,\n' % dump_bool(st.has_import_star()) # Return True if the block uses a starred from-import.
72+
out += 'Symbols: Symbols{\n'
73+
for name in st.get_identifiers():
74+
value = st.lookup(name)
75+
out += '"%s":%s,\n' % (name, dump_symbol(value))
76+
out += '},\n'
77+
# out += 'children:"%s",\n' % st.get_children() # Return a list of the nested symbol tables.
78+
out += "}"
79+
return out
80+
81+
def dump_symbol(s):
82+
"""Dump a symbol"""
83+
#class symtable.Symbol
84+
# An entry in a SymbolTable corresponding to an identifier in the source. The constructor is not public.
85+
out = "Symbol{\n"
86+
out += 'Name:"%s",\n' % s.get_name() # Return the symbol’s name.
87+
flags = []
88+
flag_bits = s._Symbol__flags
89+
for name, mask in DEF_FLAGS:
90+
if (flag_bits & mask) != 0:
91+
flags.append(name)
92+
if not flags:
93+
flags = ["0"]
94+
out += 'Flags:%s,\n' % "|".join(flags)
95+
96+
scope = SCOPES.get(s._Symbol__scope, "scopeUnknown")
97+
out += 'Scope:%s,\n' % scope
98+
99+
# Return a namespace bound to this name.
100+
ns = s.get_namespaces()
101+
if len(ns) == 0:
102+
pass
103+
elif len(ns) == 1:
104+
out += 'Namespace:%s,\n' % dump_symtable(ns[0])
105+
else:
106+
raise AssertionError("More than one namespace")
107+
out += "}"
108+
return out
109+
110+
def escape(x):
111+
"""Encode strings with backslashes for python/go"""
112+
return x.replace('\\', "\\\\").replace('"', r'\"').replace("\n", r'\n').replace("\t", r'\t')
113+
114+
def main():
115+
"""Write symtable_data_test.go"""
116+
path = "symtable_data_test.go"
117+
out = ["""// Test data generated by make_symtable_test.py - do not edit
118+
119+
package compile
120+
121+
import (
122+
"github.com/ncw/gpython/py"
123+
)
124+
125+
var symtableTestData = []struct {
126+
in string
127+
mode string // exec, eval or single
128+
out *SymTable
129+
exceptionType *py.Type
130+
errString string
131+
}{"""]
132+
for x in inp:
133+
source, mode = x[:2]
134+
if len(x) > 2:
135+
exc = x[2]
136+
try:
137+
table = symtable(source, "<string>", mode)
138+
except exc as e:
139+
error = e.msg
140+
else:
141+
raise ValueError("Expecting exception %s" % exc)
142+
table = "nil"
143+
gostring = "nil"
144+
exc_name = "py.%s" % exc.__name__
145+
else:
146+
table = symtable(source, "<string>", mode)
147+
exc_name = "nil"
148+
error = ""
149+
out.append('{"%s", "%s", %s, %s, "%s"},' % (escape(source), mode, dump_symtable(table), exc_name, escape(error)))
150+
out.append("}\n")
151+
print("Writing %s" % path)
152+
with open(path, "w") as f:
153+
f.write("\n".join(out))
154+
f.write("\n")
155+
subprocess.check_call(["gofmt", "-w", path])
156+
157+
if __name__ == "__main__":
158+
main()

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy