new test script
This commit is contained in:
ridethepig
parent
436d22b452
commit
ee09258b41
6
.vscode/settings.json
vendored
6
.vscode/settings.json
vendored
@ -10,5 +10,9 @@
|
||||
"language": "C++",
|
||||
"listeners": false,
|
||||
"visitors": true
|
||||
}
|
||||
},
|
||||
"[python]": {
|
||||
"editor.defaultFormatter": "ms-python.autopep8"
|
||||
},
|
||||
"python.formatting.provider": "none"
|
||||
}
|
||||
282
scripts/mytester.py
Normal file
282
scripts/mytester.py
Normal file
@ -0,0 +1,282 @@
|
||||
import argparse
|
||||
import re
|
||||
import os
|
||||
import subprocess
|
||||
from pretty_print import Print_C
|
||||
|
||||
# "([1-4]\d|51).*\.sy"
|
||||
|
||||
BUILD_DIR = "build/"
|
||||
LIB_PATH = "tools/sylib/libsysy_x86.a"
|
||||
HDR_PATH = "tools/sylib/sylib.h"
|
||||
|
||||
def case_collector(re_selector, dir):
|
||||
selector = re.compile(re_selector)
|
||||
testcases = []
|
||||
for filename in os.listdir(dir):
|
||||
if selector.match(filename):
|
||||
testcases.append(os.path.splitext(filename)[0])
|
||||
return sorted(testcases)
|
||||
|
||||
|
||||
|
||||
class Compiler:
|
||||
def __init__(self, scheme, target_dir, testcases):
|
||||
self.scheme = scheme
|
||||
self.testcases = testcases
|
||||
self.target_dir = target_dir
|
||||
self.sy_template = f"{target_dir}/{{testcase}}/{{testcase}}.sy"
|
||||
self.prefix = f"{target_dir}/{{testcase}}/{scheme['name']}"
|
||||
self.llvm_ir_template = f"{self.prefix}/{{testcase}}.ll"
|
||||
self.asm_template = f"{self.prefix}/{{testcase}}.s"
|
||||
self.obj_template = f"{self.prefix}/{{testcase}}.o"
|
||||
self.bin_template = f"{self.prefix}/{{testcase}}"
|
||||
self.compile_log = f"{self.prefix}/{{testcase}}_compile.log"
|
||||
self.runner_log = f"{self.prefix}/{{testcase}}_run_{{kase}}.out"
|
||||
self.myout_template = f"{self.prefix}/{{testcase}}.out"
|
||||
self.in_template = f"{target_dir}/{{testcase}}/{{testcase}}.in"
|
||||
self.count_error = 0
|
||||
|
||||
def clean_case(self, testcase):
|
||||
os.system(f"rm -rf {self.target_dir}/{testcase}/{self.scheme['name']}")
|
||||
os.makedirs(f"{self.target_dir}/{testcase}/{self.scheme['name']}")
|
||||
|
||||
def prepare_dir(self):
|
||||
for testcase in self.testcases:
|
||||
self.clean_case(testcase)
|
||||
|
||||
def sy_to_ir(self, frontend_instr, testcase):
|
||||
sy = self.sy_template.format(testcase=testcase)
|
||||
ir = self.llvm_ir_template.format(testcase=testcase)
|
||||
log = self.compile_log.format(testcase=testcase)
|
||||
log_file = open(log, "a+")
|
||||
|
||||
Print_C.print_procedure(f"Generating {ir} from {sy}")
|
||||
completed = subprocess.run(frontend_instr.format(sy=sy, ir=ir).split(), stdout=log_file, stderr=log_file, bufsize=1)
|
||||
log_file.close()
|
||||
if completed.returncode != 0:
|
||||
Print_C.print_error(f"Generate {ir} failed! See {log}")
|
||||
self.count_error += 1
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def ir_to_asm(self, ir_asm_instr, testcase):
|
||||
ir = self.llvm_ir_template.format(testcase=testcase)
|
||||
asm = self.asm_template.format(testcase=testcase)
|
||||
log = self.compile_log.format(testcase=testcase)
|
||||
|
||||
log_file = open(log, "a+")
|
||||
Print_C.print_procedure(f"Generating {asm} from {ir}")
|
||||
completed = subprocess.run(ir_asm_instr.format(ir=ir, asm=asm).split(), stdout=log_file, stderr=log_file, bufsize=1)
|
||||
log_file.close()
|
||||
if completed.returncode != 0:
|
||||
Print_C.print_error(f"Generate {asm} failed! See {log}")
|
||||
self.count_error += 1
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def asm_to_obj(self, asm_obj_instr, testcase):
|
||||
asm = self.asm_template.format(testcase=testcase)
|
||||
obj = self.obj_template.format(testcase=testcase)
|
||||
log = self.compile_log.format(testcase=testcase)
|
||||
|
||||
log_file = open(log, "a+")
|
||||
Print_C.print_procedure(f"Generating {obj} from {asm}")
|
||||
completed = subprocess.run(asm_obj_instr.format(asm=asm,obj=obj).split(), stdout=log_file, stderr=log_file, bufsize=1)
|
||||
log_file.close()
|
||||
if completed.returncode != 0:
|
||||
Print_C.print_error(f"Generate {obj} failed! See {log}")
|
||||
self.count_error += 1
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def obj_to_bin(self, obj_bin_instr, testcase):
|
||||
obj = self.obj_template.format(testcase=testcase)
|
||||
bin = self.bin_template.format(testcase=testcase)
|
||||
log = self.compile_log.format(testcase=testcase)
|
||||
|
||||
log_file = open(log, "a+")
|
||||
Print_C.print_procedure(f"Generating {bin}")
|
||||
completed = subprocess.run(obj_bin_instr.format(obj=obj,bin=bin).split(), stdout=log_file, stderr=log_file, bufsize=1)
|
||||
log_file.close()
|
||||
if completed.returncode != 0:
|
||||
Print_C.print_error(f"Generate {bin} failed! See {log}")
|
||||
self.count_error += 1
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def sy_to_asm(self, sy_asm_instr, testcase):
|
||||
asm = self.asm_template.format(testcase=testcase)
|
||||
sy = self.sy_template.format(testcase=testcase)
|
||||
log = self.compile_log.format(testcase=testcase)
|
||||
|
||||
log_file = open(log, "a+")
|
||||
Print_C.print_procedure(f"Generating {asm}")
|
||||
completed = subprocess.run(sy_asm_instr.format(asm=asm, sy=sy).split(), stdout=log_file, stderr=log_file, bufsize=1)
|
||||
log_file.close()
|
||||
if completed.returncode != 0:
|
||||
Print_C.print_error(f"Generate {bin} failed! See {log}")
|
||||
self.count_error += 1
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def compile_case(self, testcase, emit_llvm_ir=True):
|
||||
Print_C.print_subheader(f"[Compiling {self.scheme['name']} | {testcase}]")
|
||||
if emit_llvm_ir:
|
||||
if not self.sy_to_ir(self.scheme["sy_ir"], testcase=testcase): return
|
||||
if not self.ir_to_asm(self.scheme["ir_asm"], testcase=testcase): return
|
||||
else:
|
||||
if not self.sy_to_asm(self.scheme["sy_asm"], testcase=testcase): return
|
||||
if not self.asm_to_obj(self.scheme["asm_obj"], testcase=testcase): return
|
||||
if not self.obj_to_bin(self.scheme["obj_bin"], testcase=testcase): return
|
||||
|
||||
def compile_all_tests(self, error_tolerance=1, emit_llvm_ir=True):
|
||||
for testcase in self.testcases:
|
||||
self.compile_case(testcase, emit_llvm_ir)
|
||||
if self.count_error >= error_tolerance:
|
||||
Print_C.print_error(f"Test script stopped due to {self.count_error} errors")
|
||||
return
|
||||
|
||||
def run_case(self, testcase, kase=0):
|
||||
bin = self.bin_template.format(testcase=testcase)
|
||||
stdin = self.in_template.format(testcase=testcase)
|
||||
myout = self.myout_template.format(testcase=testcase)
|
||||
log = self.runner_log.format(testcase=testcase, kase=kase)
|
||||
if not os.path.exists(bin):
|
||||
Print_C.print_pass(f"testcase {testcase} not compiled, skipped")
|
||||
return False
|
||||
myout_file = open(myout, "w")
|
||||
log_file = open(log, "w")
|
||||
null_file = open(os.devnull, "w")
|
||||
|
||||
Print_C.print_procedure(f"Running {self.scheme['name']}_{testcase} [case: {kase}]")
|
||||
|
||||
if os.path.exists(stdin):
|
||||
stdin_file = open(stdin, "r")
|
||||
if kase == 0:
|
||||
p = subprocess.run(f"{bin}".split(), stdin=stdin_file, stdout=myout_file, stderr=log_file, bufsize=1)
|
||||
subprocess.run(f"echo".split(), stdout=myout_file, bufsize=1)
|
||||
subprocess.run(f"echo {p.returncode}".split(), stdout=myout_file, bufsize=1)
|
||||
else:
|
||||
p = subprocess.run(f"{bin}".split(), stdin=stdin_file, stdout=null_file, stderr=null_file, bufsize=1)
|
||||
stdin_file.close()
|
||||
else:
|
||||
if kase == 0:
|
||||
p = subprocess.run(f"{bin}".split(), stdout=myout_file, stderr=log_file, bufsize=1)
|
||||
subprocess.run(f"echo".split(), stdout=myout_file, bufsize=1)
|
||||
subprocess.run(f"echo {p.returncode}".split(), stdout=myout_file, bufsize=1)
|
||||
else:
|
||||
p = subprocess.run(f"{bin}".split(), stdout=null_file, stderr=null_file, bufsize=1)
|
||||
Print_C.print_pass(f" {self.scheme['name']}_{testcase} [case: {kase}] exit with {p.returncode}")
|
||||
myout_file.close()
|
||||
log_file.close()
|
||||
return True
|
||||
|
||||
def run_all_tests(self, run_kases=1):
|
||||
for kase in range(run_kases):
|
||||
Print_C.print_subheader(f"[Running CASE {kase}]")
|
||||
for testcase in self.testcases:
|
||||
self.run_case(testcase=testcase, kase=kase)
|
||||
|
||||
class BatTest:
|
||||
def __init__(self, compiler, compiler_rf, testcases):
|
||||
self.compiler = compiler
|
||||
self.compiler_rf = compiler_rf
|
||||
self.testcases = testcases
|
||||
self.count_error = 0
|
||||
self.label = f"{compiler.scheme['name']}_{compiler_rf.scheme['name']}"
|
||||
self.diffout_template = f"{compiler.target_dir}/{{testcase}}/{self.label}.diff"
|
||||
|
||||
def bat_case(self, testcase, compile=False):
|
||||
if compile:
|
||||
self.compiler.clean_case(testcase)
|
||||
self.compiler_rf.clean_case(testcase)
|
||||
self.compiler.compile_case(testcase)
|
||||
self.compiler_rf.compile_case(testcase)
|
||||
if not compiler.run_case(testcase, 0): return False
|
||||
if not compiler_rf.run_case(testcase, 0): return False
|
||||
myout = compiler.myout_template.format(testcase=testcase)
|
||||
refout = compiler_rf.myout_template.format(testcase=testcase)
|
||||
diffout = self.diffout_template.format(testcase=testcase)
|
||||
diffout_file = open(diffout, "w")
|
||||
p = subprocess.run(f"diff {myout} {refout}".split(), stdout=diffout_file, stderr=diffout_file, bufsize=1)
|
||||
diffout_file.close()
|
||||
if p.returncode != 0:
|
||||
Print_C.print_error(f"Different output for {testcase}. See {diffout}")
|
||||
return False
|
||||
return True
|
||||
|
||||
def bat_all_tests(self, compile=False):
|
||||
for testcase in self.testcases:
|
||||
if not self.bat_case(testcase, compile): break
|
||||
|
||||
scheme_ref = {
|
||||
"name": "reference",
|
||||
"sy_ir": f"clang -x c -c -fPIE -m32 -S -emit-llvm -include {HDR_PATH} {{sy}} -o {{ir}}",
|
||||
"ir_asm": "llc --march=x86 --relocation-model=pic {ir} -o {asm}",
|
||||
"asm_obj": "as --32 {asm} -o {obj}",
|
||||
"obj_bin": f"clang -m32 -Ofast -fPIE {{obj}} {LIB_PATH} -o {{bin}}",
|
||||
"sy_asm": f"clang -x c -c -fPIE -m32 -S -include {HDR_PATH} {{sy}} -o {{asm}}"
|
||||
}
|
||||
|
||||
scheme_my = {
|
||||
"name": "my",
|
||||
"sy_ir": f"build/sysy -S {{sy}} -o {{ir}}",
|
||||
"ir_asm": "llc --march=x86 --relocation-model=pic {ir} -o {asm}",
|
||||
"asm_obj": "as --32 {asm} -o {obj}",
|
||||
"obj_bin": f"clang -m32 -Ofast -fPIE {{obj}} {LIB_PATH} -o {{bin}}",
|
||||
"sy_asm": f"clang -x c -c -fPIE -m32 -S -include {HDR_PATH} {{sy}} -o {{asm}}"
|
||||
}
|
||||
|
||||
schemes = {
|
||||
"my": scheme_my,
|
||||
"reference": scheme_ref
|
||||
}
|
||||
# subprocess.run(f"llc -O3 -march=arm -mcpu=cortex-a72 -float-abi=hard -filetype=asm {ir} -o {asm}".split(), stdout=log_file, stderr=log_file, bufsize=1)
|
||||
# subprocess.run(f"as -march=armv7-a -mfloat-abi=hard {asm} -o {obj}".split(), stdout=log_file, stderr=log_file, bufsize=1)
|
||||
# subprocess.run(f"clang -Ofast -marm -march=armv7-a -mfpu=neon -mfloat-abi=hard {obj} {lib} -o {bin}".split(), stdout=log_file, stderr=log_file, bufsize=1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(prog="SysY test script")
|
||||
parser.add_argument('--scheme', default='reference')
|
||||
parser.add_argument('--case_prefix', default='testcases/functional')
|
||||
parser.add_argument('--case_selector', default='.*\.sy')
|
||||
parser.add_argument("--clean", action='store_true', default=False)
|
||||
parser.add_argument('--compile', action='store_true', default=False)
|
||||
parser.add_argument('--run', action='store_true', default=False)
|
||||
parser.add_argument('--bat', action='store_true', default=False)
|
||||
args = parser.parse_args()
|
||||
print(args)
|
||||
testcases = case_collector(args.case_selector, args.case_prefix)
|
||||
target_dir = os.path.join(BUILD_DIR, args.case_prefix)
|
||||
if args.clean:
|
||||
os.system(f"rm -rfI {target_dir}")
|
||||
if not os.path.exists(target_dir):
|
||||
os.makedirs(target_dir)
|
||||
for testcase in testcases:
|
||||
testcase_dir = os.path.join(target_dir, testcase)
|
||||
testcase_src = os.path.join(args.case_prefix, testcase)
|
||||
if not os.path.exists(testcase_dir):
|
||||
os.makedirs(testcase_dir)
|
||||
os.system(f"cp {testcase_src}.* {testcase_dir}/")
|
||||
current_schem = schemes[args.scheme]
|
||||
if args.bat:
|
||||
compiler = Compiler(scheme=current_schem,target_dir=target_dir, testcases=testcases)
|
||||
compiler_rf = Compiler(scheme=schemes["reference"],target_dir=target_dir, testcases=testcases)
|
||||
battester = BatTest(compiler, compiler_rf, testcases)
|
||||
battester.bat_all_tests(args.compile)
|
||||
elif args.compile:
|
||||
compiler = Compiler(scheme=current_schem,target_dir=target_dir, testcases=testcases)
|
||||
compiler.prepare_dir()
|
||||
compiler.compile_all_tests(error_tolerance=1, emit_llvm_ir=True)
|
||||
if args.run:
|
||||
compiler.run_all_tests()
|
||||
elif args.run:
|
||||
compiler = Compiler(scheme=current_schem,target_dir=target_dir, testcases=testcases)
|
||||
compiler.run_all_tests()
|
||||
Loading…
Reference in New Issue
Block a user