new test script

.vscode/settings.json

@@ -10,5 +10,9 @@
         "language": "C++",
         "listeners": false,
         "visitors": true
-    }
+    },
+    "[python]": {
+        "editor.defaultFormatter": "ms-python.autopep8"
+    },
+    "python.formatting.provider": "none"
 }

scripts/mytester.py

@@ -0,0 +1,282 @@
+import argparse
+import re
+import os
+import subprocess
+from pretty_print import Print_C
+
+# "([1-4]\d|51).*\.sy"
+
+BUILD_DIR = "build/"
+LIB_PATH = "tools/sylib/libsysy_x86.a"
+HDR_PATH = "tools/sylib/sylib.h"
+
+def case_collector(re_selector, dir):
+    selector = re.compile(re_selector)
+    testcases = []
+    for filename in os.listdir(dir):
+        if selector.match(filename):
+            testcases.append(os.path.splitext(filename)[0])
+    return sorted(testcases)
+
+
+
+class Compiler:
+    def __init__(self, scheme, target_dir, testcases):
+        self.scheme = scheme
+        self.testcases = testcases
+        self.target_dir = target_dir
+        self.sy_template = f"{target_dir}/{{testcase}}/{{testcase}}.sy"
+        self.prefix = f"{target_dir}/{{testcase}}/{scheme['name']}"
+        self.llvm_ir_template = f"{self.prefix}/{{testcase}}.ll"
+        self.asm_template = f"{self.prefix}/{{testcase}}.s"
+        self.obj_template = f"{self.prefix}/{{testcase}}.o"
+        self.bin_template = f"{self.prefix}/{{testcase}}"
+        self.compile_log = f"{self.prefix}/{{testcase}}_compile.log"
+        self.runner_log = f"{self.prefix}/{{testcase}}_run_{{kase}}.out"
+        self.myout_template = f"{self.prefix}/{{testcase}}.out"
+        self.in_template = f"{target_dir}/{{testcase}}/{{testcase}}.in"
+        self.count_error = 0
+
+    def clean_case(self, testcase):
+        os.system(f"rm -rf {self.target_dir}/{testcase}/{self.scheme['name']}")
+        os.makedirs(f"{self.target_dir}/{testcase}/{self.scheme['name']}")
+
+    def prepare_dir(self):
+        for testcase in self.testcases:
+            self.clean_case(testcase)
+
+    def sy_to_ir(self, frontend_instr, testcase):
+        sy = self.sy_template.format(testcase=testcase)
+        ir = self.llvm_ir_template.format(testcase=testcase)
+        log = self.compile_log.format(testcase=testcase)
+        log_file = open(log, "a+")
+
+        Print_C.print_procedure(f"Generating {ir} from {sy}")
+        completed = subprocess.run(frontend_instr.format(sy=sy, ir=ir).split(), stdout=log_file, stderr=log_file, bufsize=1)
+        log_file.close()
+        if completed.returncode != 0:
+            Print_C.print_error(f"Generate {ir} failed! See {log}")
+            self.count_error += 1
+            return False
+        return True
+
+
+    def ir_to_asm(self, ir_asm_instr, testcase):
+        ir = self.llvm_ir_template.format(testcase=testcase)
+        asm = self.asm_template.format(testcase=testcase)
+        log = self.compile_log.format(testcase=testcase)
+
+        log_file = open(log, "a+")
+        Print_C.print_procedure(f"Generating {asm} from {ir}")
+        completed = subprocess.run(ir_asm_instr.format(ir=ir, asm=asm).split(), stdout=log_file, stderr=log_file, bufsize=1)
+        log_file.close()
+        if completed.returncode != 0:
+            Print_C.print_error(f"Generate {asm} failed! See {log}")
+            self.count_error += 1
+            return False
+        return True
+
+
+    def asm_to_obj(self, asm_obj_instr, testcase):
+        asm = self.asm_template.format(testcase=testcase)
+        obj = self.obj_template.format(testcase=testcase)
+        log = self.compile_log.format(testcase=testcase)
+
+        log_file = open(log, "a+")
+        Print_C.print_procedure(f"Generating {obj} from {asm}")
+        completed = subprocess.run(asm_obj_instr.format(asm=asm,obj=obj).split(), stdout=log_file, stderr=log_file, bufsize=1)
+        log_file.close()
+        if completed.returncode != 0:
+            Print_C.print_error(f"Generate {obj} failed! See {log}")
+            self.count_error += 1
+            return False
+        return True
+
+
+    def obj_to_bin(self, obj_bin_instr, testcase):
+        obj = self.obj_template.format(testcase=testcase)
+        bin = self.bin_template.format(testcase=testcase)
+        log = self.compile_log.format(testcase=testcase)
+
+        log_file = open(log, "a+")
+        Print_C.print_procedure(f"Generating {bin}")
+        completed = subprocess.run(obj_bin_instr.format(obj=obj,bin=bin).split(), stdout=log_file, stderr=log_file, bufsize=1)
+        log_file.close()
+        if completed.returncode != 0:
+            Print_C.print_error(f"Generate {bin} failed! See {log}")
+            self.count_error += 1
+            return False
+        return True
+
+
+    def sy_to_asm(self, sy_asm_instr, testcase):
+        asm = self.asm_template.format(testcase=testcase)
+        sy = self.sy_template.format(testcase=testcase)
+        log = self.compile_log.format(testcase=testcase)
+
+        log_file = open(log, "a+")
+        Print_C.print_procedure(f"Generating {asm}")
+        completed = subprocess.run(sy_asm_instr.format(asm=asm, sy=sy).split(), stdout=log_file, stderr=log_file, bufsize=1)
+        log_file.close()
+        if completed.returncode != 0:
+            Print_C.print_error(f"Generate {bin} failed! See {log}")
+            self.count_error += 1
+            return False
+        return True
+
+
+    def compile_case(self, testcase, emit_llvm_ir=True):
+        Print_C.print_subheader(f"[Compiling {self.scheme['name']} | {testcase}]")
+        if emit_llvm_ir:
+            if not self.sy_to_ir(self.scheme["sy_ir"], testcase=testcase): return
+            if not self.ir_to_asm(self.scheme["ir_asm"], testcase=testcase): return
+        else:
+            if not self.sy_to_asm(self.scheme["sy_asm"], testcase=testcase): return
+        if not self.asm_to_obj(self.scheme["asm_obj"], testcase=testcase): return
+        if not self.obj_to_bin(self.scheme["obj_bin"], testcase=testcase): return
+    
+    def compile_all_tests(self, error_tolerance=1, emit_llvm_ir=True):
+        for testcase in self.testcases:
+            self.compile_case(testcase, emit_llvm_ir)
+            if self.count_error >= error_tolerance:
+                Print_C.print_error(f"Test script stopped due to {self.count_error} errors")
+                return
+
+    def run_case(self, testcase, kase=0):
+        bin = self.bin_template.format(testcase=testcase)
+        stdin = self.in_template.format(testcase=testcase)
+        myout = self.myout_template.format(testcase=testcase)
+        log = self.runner_log.format(testcase=testcase, kase=kase)
+        if not os.path.exists(bin):
+            Print_C.print_pass(f"testcase {testcase} not compiled, skipped")
+            return False
+        myout_file = open(myout, "w")
+        log_file = open(log, "w")
+        null_file = open(os.devnull, "w")
+
+        Print_C.print_procedure(f"Running {self.scheme['name']}_{testcase} [case: {kase}]")
+
+        if os.path.exists(stdin):
+            stdin_file = open(stdin, "r")
+            if kase == 0:
+                p = subprocess.run(f"{bin}".split(), stdin=stdin_file, stdout=myout_file, stderr=log_file, bufsize=1)
+                subprocess.run(f"echo".split(), stdout=myout_file, bufsize=1)
+                subprocess.run(f"echo {p.returncode}".split(), stdout=myout_file, bufsize=1)
+            else:
+                p = subprocess.run(f"{bin}".split(), stdin=stdin_file, stdout=null_file, stderr=null_file, bufsize=1)
+            stdin_file.close()
+        else:
+            if kase == 0:
+                p = subprocess.run(f"{bin}".split(), stdout=myout_file, stderr=log_file, bufsize=1)
+                subprocess.run(f"echo".split(), stdout=myout_file, bufsize=1)
+                subprocess.run(f"echo {p.returncode}".split(), stdout=myout_file, bufsize=1)
+            else:
+                p = subprocess.run(f"{bin}".split(), stdout=null_file, stderr=null_file, bufsize=1)
+        Print_C.print_pass(f"  {self.scheme['name']}_{testcase} [case: {kase}] exit with {p.returncode}")
+        myout_file.close()
+        log_file.close()
+        return True
+
+    def run_all_tests(self, run_kases=1):
+        for kase in range(run_kases):
+            Print_C.print_subheader(f"[Running CASE {kase}]")
+            for testcase in self.testcases:
+                self.run_case(testcase=testcase, kase=kase)
+
+class BatTest:
+    def __init__(self, compiler, compiler_rf, testcases):
+        self.compiler = compiler
+        self.compiler_rf = compiler_rf
+        self.testcases = testcases
+        self.count_error = 0
+        self.label = f"{compiler.scheme['name']}_{compiler_rf.scheme['name']}"
+        self.diffout_template = f"{compiler.target_dir}/{{testcase}}/{self.label}.diff"
+
+    def bat_case(self, testcase, compile=False):
+        if compile:
+            self.compiler.clean_case(testcase)
+            self.compiler_rf.clean_case(testcase)
+            self.compiler.compile_case(testcase)
+            self.compiler_rf.compile_case(testcase)
+        if not compiler.run_case(testcase, 0): return False
+        if not compiler_rf.run_case(testcase, 0): return False
+        myout = compiler.myout_template.format(testcase=testcase)
+        refout = compiler_rf.myout_template.format(testcase=testcase)
+        diffout = self.diffout_template.format(testcase=testcase)
+        diffout_file = open(diffout, "w")
+        p = subprocess.run(f"diff {myout} {refout}".split(), stdout=diffout_file, stderr=diffout_file, bufsize=1)
+        diffout_file.close()
+        if p.returncode != 0:
+            Print_C.print_error(f"Different output for {testcase}. See {diffout}")
+            return False
+        return True
+
+    def bat_all_tests(self, compile=False):
+        for testcase in self.testcases:
+            if not self.bat_case(testcase, compile): break
+
+scheme_ref = {
+    "name": "reference",
+    "sy_ir": f"clang -x c -c -fPIE -m32 -S -emit-llvm -include {HDR_PATH} {{sy}} -o {{ir}}",
+    "ir_asm": "llc --march=x86 --relocation-model=pic {ir} -o {asm}",
+    "asm_obj": "as --32 {asm} -o {obj}",
+    "obj_bin": f"clang -m32 -Ofast -fPIE {{obj}} {LIB_PATH} -o {{bin}}",
+    "sy_asm": f"clang -x c -c -fPIE -m32 -S -include {HDR_PATH} {{sy}} -o {{asm}}"
+}
+
+scheme_my = {
+    "name": "my",
+    "sy_ir": f"build/sysy -S {{sy}} -o {{ir}}",
+    "ir_asm": "llc --march=x86 --relocation-model=pic {ir} -o {asm}",
+    "asm_obj": "as --32 {asm} -o {obj}",
+    "obj_bin": f"clang -m32 -Ofast -fPIE {{obj}} {LIB_PATH} -o {{bin}}",
+    "sy_asm": f"clang -x c -c -fPIE -m32 -S -include {HDR_PATH} {{sy}} -o {{asm}}"
+}
+
+schemes = {
+    "my": scheme_my,
+    "reference": scheme_ref
+}
+# subprocess.run(f"llc -O3 -march=arm -mcpu=cortex-a72 -float-abi=hard -filetype=asm {ir} -o {asm}".split(), stdout=log_file, stderr=log_file, bufsize=1)
+# subprocess.run(f"as -march=armv7-a -mfloat-abi=hard {asm} -o {obj}".split(), stdout=log_file, stderr=log_file, bufsize=1)
+# subprocess.run(f"clang -Ofast -marm -march=armv7-a -mfpu=neon -mfloat-abi=hard {obj} {lib} -o {bin}".split(), stdout=log_file, stderr=log_file, bufsize=1)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(prog="SysY test script")
+    parser.add_argument('--scheme', default='reference')
+    parser.add_argument('--case_prefix', default='testcases/functional')
+    parser.add_argument('--case_selector', default='.*\.sy')
+    parser.add_argument("--clean", action='store_true', default=False)
+    parser.add_argument('--compile', action='store_true', default=False)
+    parser.add_argument('--run', action='store_true', default=False)
+    parser.add_argument('--bat', action='store_true', default=False)
+    args = parser.parse_args()
+    print(args)
+    testcases = case_collector(args.case_selector, args.case_prefix)
+    target_dir = os.path.join(BUILD_DIR, args.case_prefix)
+    if args.clean:
+        os.system(f"rm -rfI {target_dir}")
+    if not os.path.exists(target_dir):
+        os.makedirs(target_dir)
+    for testcase in testcases:
+        testcase_dir = os.path.join(target_dir, testcase)
+        testcase_src = os.path.join(args.case_prefix, testcase)
+        if not os.path.exists(testcase_dir):
+            os.makedirs(testcase_dir)
+            os.system(f"cp {testcase_src}.* {testcase_dir}/")
+    current_schem = schemes[args.scheme]
+    if args.bat:
+        compiler = Compiler(scheme=current_schem,target_dir=target_dir, testcases=testcases)
+        compiler_rf = Compiler(scheme=schemes["reference"],target_dir=target_dir, testcases=testcases)
+        battester = BatTest(compiler, compiler_rf, testcases)
+        battester.bat_all_tests(args.compile)
+    elif args.compile:
+        compiler = Compiler(scheme=current_schem,target_dir=target_dir, testcases=testcases)
+        compiler.prepare_dir()
+        compiler.compile_all_tests(error_tolerance=1, emit_llvm_ir=True)
+        if args.run:
+            compiler.run_all_tests()
+    elif args.run:
+        compiler = Compiler(scheme=current_schem,target_dir=target_dir, testcases=testcases)
+        compiler.run_all_tests()