infrastructure update

.gitignore

@@ -1,3 +1,4 @@
 build/
 .antlr/
-*.log
+*.log
+__pycache__/

include/common.h

@@ -15,7 +15,7 @@
 #define sysy_assert(cond)                                                                                              \
   do {                                                                                                                 \
     if (!(cond))                                                                                                       \
-      throw  GrammarException(__FILE__, __LINE__, #cond);                                                                     \
+      throw GrammarException(__FILE__, __LINE__, #cond);                                                               \
   } while (0)
 
 namespace antlrSysY {

scripts/analyzer.py

@@ -0,0 +1,47 @@
+import os
+import subprocess
+from pretty_print import Print_C
+
+class Analyzer:
+    def __init__(self, scheme, emit_llvm_ir, testcases) -> None:
+        self.scheme = scheme
+        self.emit_llvm_ir = emit_llvm_ir
+        self.testcases = testcases
+
+        self.asm_template = f"build/test_results/{{testcase}}/asm/{scheme}.s"
+        self.perf_template = f"build/test_results/{{testcase}}/analyze/perf/{scheme}.txt"
+        self.llvm_mca_template = f"build/test_results/{{testcase}}/analyze/llvm_mca/{scheme}.txt"
+
+        for testcase in testcases:
+            self.__generate_path(testcase)
+
+    def __generate_path(self, testcase):
+        llvm_mca_path = f"build/test_results/{testcase}/analyze/llvm_mca/"
+        perf_path = f"build/test_results/{testcase}/analyze/perf/"
+
+        if not os.path.exists(llvm_mca_path):
+            os.makedirs(llvm_mca_path)
+
+        if not os.path.exists(perf_path):
+            os.makedirs(perf_path)
+
+    def run_perf(self): # TODO
+        pass
+
+    def run_llvm_mca(self, testcase):
+        asm = self.asm_template.format(testcase=testcase)
+        llvm_mca = self.llvm_mca_template.format(testcase=testcase)
+
+        llvm_mca_file = open(llvm_mca, "w+")
+
+        subprocess.run(f"llvm-mca -mtriple=armv7 -mcpu=cortex-a72 -timeline {asm}".split(), stdout=llvm_mca_file, bufsize=1)
+
+        llvm_mca_file.close()
+
+
+    def analyze(self):
+        for testcase in self.testcases:
+            Print_C.print_subheader(f"[Analyzing {self.scheme} | {testcase}]")
+            # self.run_llvm_mca(testcase=testcase)
+
+            # self.run_perf()

scripts/compiler.py

@@ -0,0 +1,146 @@
+import os
+import subprocess
+from pretty_print import Print_C
+
+lib = "build/lib/libsysy.a"
+header = "build/include/sylib.h"
+
+class Compiler:
+    def __init__(self, scheme, testcases):
+        self.scheme = scheme
+        self.testcases = testcases
+
+        self.sy_template = f"testcases/{{testcase}}.sy"
+        self.sy_fix_template = f"testcases/{{testcase}}.{scheme}_sy"
+        self.sy_category_template = f"testcases/{{category}}/{{testcase}}.sy"
+        self.llvm_ir_template = f"build/test_results/{{testcase}}/llvm_ir/{scheme}.ll"
+        self.asm_template = f"build/test_results/{{testcase}}/asm/{scheme}.s"
+        self.obj_template = f"build/test_results/{{testcase}}/obj/{scheme}.o"
+        self.bin_template = f"build/test_results/{{testcase}}/bin/{scheme}"
+        self.compile_log = f"build/log/compile_log/{{testcase}}/{scheme}.log"
+
+        for testcase in testcases:
+            self.__generate_path(testcase)
+
+
+    def __generate_path(self, testcase):
+        llvm_ir_path = f"build/test_results/{testcase}/llvm_ir/"
+        asm_path = f"build/test_results/{testcase}/asm/"
+        obj_path = f"build/test_results/{testcase}/obj/"
+        bin_path = f"build/test_results/{testcase}/bin/"
+        compile_log_path = f"build/log/compile_log/{testcase}/"
+
+        if not os.path.exists(llvm_ir_path):
+            os.makedirs(llvm_ir_path)
+
+        if not os.path.exists(asm_path):
+            os.makedirs(asm_path)
+
+        if not os.path.exists(obj_path):
+            os.makedirs(obj_path)
+
+        if not os.path.exists(bin_path):
+            os.makedirs(bin_path)
+
+        if not os.path.exists(compile_log_path):
+            os.makedirs(compile_log_path)
+
+
+    def sy_to_ir(self, frontend_instr, testcase, category=""):
+        sy = self.sy_template.format(testcase=testcase)
+        sy_fix = self.sy_fix_template.format(testcase=testcase)
+        sy_category = self.sy_category_template.format(testcase=testcase, category=category)
+        if os.path.exists(sy_fix):
+            sy = sy_fix
+        elif os.path.exists(sy_category):
+            sy = sy_category
+
+        ir = self.llvm_ir_template.format(testcase=testcase)
+        log = self.compile_log.format(testcase=testcase)
+
+        log_file = open(log, "a+")
+
+        Print_C.print_procedure(f"Generating {self.scheme}.ll")
+        completed = subprocess.run(frontend_instr.format(header=header, sy=sy, ir=ir).split(), stdout=log_file, stderr=log_file, bufsize=1)
+        if completed.returncode != 0:
+            Print_C.print_error(f"Generating {self.scheme}.ll failed! See {log}")
+        log_file.close()
+
+
+    def ir_to_asm(self, testcase):
+        ir = self.llvm_ir_template.format(testcase=testcase)
+        asm = self.asm_template.format(testcase=testcase)
+        log = self.compile_log.format(testcase=testcase)
+
+        log_file = open(log, "a+")
+
+        Print_C.print_procedure(f"Generating {self.scheme}.s")
+        subprocess.run(f"llc -O3 -march=arm -mcpu=cortex-a72 -float-abi=hard -filetype=asm {ir} -o {asm}".split(), stdout=log_file, stderr=log_file, bufsize=1)
+
+        log_file.close()
+
+
+    def asm_to_obj(self, testcase):
+        asm = self.asm_template.format(testcase=testcase)
+        obj = self.obj_template.format(testcase=testcase)
+        log = self.compile_log.format(testcase=testcase)
+
+        log_file = open(log, "a+")
+
+        Print_C.print_procedure(f"Generating {self.scheme}.o")
+        subprocess.run(f"as -march=armv7-a -mfloat-abi=hard {asm} -o {obj}".split(), stdout=log_file, stderr=log_file, bufsize=1)
+
+        log_file.close()
+
+    def obj_to_bin(self, testcase):
+        obj = self.obj_template.format(testcase=testcase)
+        bin = self.bin_template.format(testcase=testcase)
+        log = self.compile_log.format(testcase=testcase)
+
+        log_file = open(log, "a+")
+
+        Print_C.print_procedure(f"Generating {self.scheme}")
+        subprocess.run(f"clang -Ofast -marm -march=armv7-a -mfpu=neon -mfloat-abi=hard {obj} {lib} -o {bin}".split(), stdout=log_file, stderr=log_file, bufsize=1)
+
+        log_file.close()
+
+
+    def sy_to_asm(self, frontend_instr, testcase):
+        asm = self.asm_template.format(testcase=testcase)
+        sy = self.sy_template.format(testcase=testcase)
+        sy_fix = self.sy_fix_template.format(testcase=testcase)
+        if os.path.exists(sy_fix):
+            sy = sy_fix
+
+        log = self.compile_log.format(testcase=testcase)
+
+        log_file = open(log, "a+")
+
+        Print_C.print_procedure(f"Generating {self.scheme}.s")
+        subprocess.run(frontend_instr.format(header=header, asm=asm, sy=sy).split(), stdout=log_file, stderr=log_file, bufsize=1)
+
+
+        log_file.close()
+
+
+    def compile_all_tests(self, frontend_instr, emit_llvm_ir):
+        if emit_llvm_ir:
+            for testcase in self.testcases:
+                Print_C.print_subheader(f"[Compiling {self.scheme} | {testcase}]")
+                self.sy_to_ir(frontend_instr=frontend_instr, testcase=testcase)
+                self.ir_to_asm(testcase=testcase)
+                self.asm_to_obj(testcase=testcase)
+                self.obj_to_bin(testcase=testcase)
+        else:
+            for testcase in self.testcases:
+                Print_C.print_subheader(f"[Compiling {self.scheme} | {testcase}]")
+                self.sy_to_asm(frontend_instr=frontend_instr, testcase=testcase)
+                self.asm_to_obj(testcase=testcase)
+                self.obj_to_bin(testcase=testcase)
+
+    def compile_rubbish(self, frontend_instr, category):
+        for testcase in self.testcases:
+            Print_C.print_subheader(f"[Compiling {self.scheme} | {testcase}]")
+            self.sy_to_ir(frontend_instr=frontend_instr, testcase=testcase, category=category)
+
+        

scripts/presenter.py

@@ -0,0 +1,87 @@
+import re
+from runner import Runner
+from collections import Counter
+from pretty_print import Print_C
+
+class Presenter:
+    kases = Runner.run_kases
+    splitter = re.compile('[-@ :\t\r\n]+')
+    scheme_time_print_template = "{0:<10} :  {1:<10.0f}us"
+
+    def __init__(self, schemes, testcases):
+        self.schemes = schemes
+        self.testcases = testcases
+
+    def __diff(self, file1, file2):
+        file1_lines = open(file1).readlines()
+        file2_lines = open(file2).readlines()
+        return file1_lines == file2_lines
+
+    def __time_to_us(self, l):
+        hours = int(l[0][:-1])
+        minutes = int(l[1][:-1])
+        seconds = int(l[2][:-1])
+        us = int(l[3][:-2])
+        return hours * 60 * 60 * 1000000 + minutes * 60 * 1000000 + seconds * 1000000 + us
+
+    def present_single_testcase(self, testcase):
+        Print_C.print_subheader(f"[Checking {testcase}]")
+
+        scheme_time_counter = Counter({scheme: 0 for scheme in self.schemes}) # {scheme: time}
+        stage_scheme_time_dict = {}
+        wrong_schemes = []
+        for scheme in self.schemes:
+            stdout = f"testcases/{testcase}.out"
+            myout = f"build/output/{testcase}/{scheme}.out"
+            if not self.__diff(stdout, myout):
+                wrong_schemes.append(scheme)
+                continue
+
+            stage_time_counter = Counter({}) # {stage: time}
+            for kase in range(Presenter.kases):
+                runner_log = f"build/log/run_log/{testcase}/{scheme}_{kase}.out"
+                lines = open(runner_log).readlines()
+                # ['Timer', 'startLine', 'endLine', 'xH', 'xM', 'xS', 'xus'][1:]
+                parsed_lines = [Presenter.splitter.split(line)[1:] for line in lines][:-1]
+                stage_time = {f"{int(line[0])} ~ {int(line[1])}": self.__time_to_us(line[2:]) for line in parsed_lines}
+                stage_time_counter += Counter(stage_time)
+
+            for stage_time in stage_time_counter.items():
+                if not stage_time[0] in stage_scheme_time_dict:
+                    stage_scheme_time_dict[stage_time[0]] = {}
+                stage_scheme_time_dict[stage_time[0]][scheme] = stage_time[1] / Presenter.kases
+
+            scheme_time_counter[scheme] = sum(stage_time_counter.values()) / Presenter.kases
+
+        Print_C.print_procedure("[AVERAGE TIME COUNT (sorted)]")
+        sorted_scheme_time_list = sorted(scheme_time_counter.items(), key=lambda scheme_time: scheme_time[1])
+        for scheme_time in sorted_scheme_time_list:
+            if not scheme_time[0] in wrong_schemes:
+                Print_C.print_pass(self.scheme_time_print_template.format(scheme_time[0], scheme_time[1]))
+        for scheme in  wrong_schemes:
+            Print_C.print_error(f"{scheme} WA")
+
+        for stage_scheme_time in stage_scheme_time_dict.items():
+            Print_C.print_procedure(f"[Line {stage_scheme_time[0]}]")
+
+            sorted_stage_scheme_list = sorted(stage_scheme_time[1].items(), key=lambda scheme_time: scheme_time[1])
+            for scheme_time in sorted_stage_scheme_list:
+                print(self.scheme_time_print_template.format(scheme_time[0], scheme_time[1]))
+
+        return scheme_time_counter
+
+    def present_all_testcases(self):
+        Print_C.print_header(f"[Checking & Racing]")
+        scheme_tot_time_counter = Counter({scheme: 0 for scheme in self.schemes}) # {scheme: total_time}
+
+        for testcase in self.testcases:
+            scheme_tot_time_counter += Counter(self.present_single_testcase(testcase=testcase))
+            print()
+
+        print()
+        sorted_scheme_tot_time_list = sorted(scheme_tot_time_counter.items(), key=lambda scheme_tot_time: scheme_tot_time[1])
+        Print_C.print_header(f"[TOTAL TIME RANKING]")
+        for scheme_tot_time in sorted_scheme_tot_time_list:
+            print(self.scheme_time_print_template.format(scheme_tot_time[0], scheme_tot_time[1]))
+        print()
+        print()

scripts/pretty_print.py

@@ -0,0 +1,40 @@
+import os
+class Print_C:
+    GREEN = '\033[32m'
+    RED = '\033[31m'
+    YELLOW = '\033[33m'
+    BLUE = '\033[34m'
+    BOLD = '\033[1m'
+    ENDC = '\033[0m'
+    if not os.path.exists("build/log"):
+        os.makedirs("build/log")
+    log = "build/log/test_result.log"
+    log_file = open(log, "a+")
+
+    def print_error(msg):
+        content = f"{Print_C.RED}{msg}{Print_C.ENDC}"
+        print(content)
+        Print_C.log_file.write(content + "\n")
+
+
+    def print_header(msg):
+        content = f"{Print_C.GREEN}{Print_C.BOLD}{msg}{Print_C.ENDC}"
+        print(content)
+        Print_C.log_file.write(content + "\n")
+
+    def print_pass(msg):
+        content = f"{Print_C.GREEN}{msg}{Print_C.ENDC}"
+        print(content)
+        Print_C.log_file.write(content + "\n")
+
+
+    def print_subheader(msg):
+        content = f"{Print_C.YELLOW}{msg}{Print_C.ENDC}"
+        print(content)
+        Print_C.log_file.write(content + "\n")
+
+
+    def print_procedure(msg):
+        content = f"{Print_C.BLUE}{msg}{Print_C.ENDC}"
+        print(content)
+        Print_C.log_file.write(content + "\n")

scripts/runner.py

@@ -0,0 +1,69 @@
+import os
+import subprocess
+from pretty_print import Print_C
+
+class Runner:
+    run_kases = 3
+
+    def __init__(self, scheme, testcases):
+        self.scheme = scheme
+        self.testcases = testcases
+
+        self.bin_file_template = f"build/test_results/{{testcase}}/bin/{scheme}"
+        self.myout_template = f"build/output/{{testcase}}/{scheme}.out"
+        self.runner_log = f"build/log/run_log/{{testcase}}/{scheme}_{{kase}}.out"
+
+        for testcase in testcases:
+            self.__generate_path(testcase)
+
+
+    def __generate_path(self, testcase):
+        myout_path = f"build/output/{testcase}/"
+        runner_log_path = f"build/log/run_log/{testcase}/"
+
+        if not os.path.exists(myout_path):
+            os.makedirs(myout_path)
+
+        if not os.path.exists(runner_log_path):
+            os.makedirs(runner_log_path)
+
+
+    def run_single_test(self, testcase, kase):
+        bin = self.bin_file_template.format(testcase=testcase)
+        stdin = f"testcases/{testcase}.in"
+        myout = self.myout_template.format(testcase=testcase)
+        log = self.runner_log.format(testcase=testcase, kase=kase)
+
+        myout_file = open(myout, "a+")
+        log_file = open(log, "a+")
+        null_file = open(os.devnull, "w")
+
+        Print_C.print_procedure(f"Running {self.scheme}_{testcase} [kase: {kase}]")
+
+        if os.path.exists(stdin):
+            stdin_file = open(stdin, "r")
+            if kase == 0:
+                p = subprocess.run(f"{bin}".split(), stdin=stdin_file, stdout=myout_file, stderr=log_file, bufsize=1)
+                subprocess.run(f"echo".split(), stdout=myout_file, bufsize=1)
+                subprocess.run(f"echo {p.returncode}".split(), stdout=myout_file, bufsize=1)
+            else:
+                p = subprocess.run(f"{bin}".split(), stdin=stdin_file, stdout=null_file, stderr=null_file, bufsize=1)
+            stdin_file.close()
+        else:
+            if kase == 0:
+                p = subprocess.run(f"{bin}".split(), stdout=myout_file, stderr=log_file, bufsize=1)
+                subprocess.run(f"echo".split(), stdout=myout_file, bufsize=1)
+                subprocess.run(f"echo {p.returncode}".split(), stdout=myout_file, bufsize=1)
+            else:
+                p = subprocess.run(f"{bin}".split(), stdout=null_file, stderr=null_file, bufsize=1)
+
+        myout_file.close()
+        log_file.close()
+
+
+    def run_all_tests(self):
+        for kase in range(Runner.run_kases):
+            Print_C.print_subheader(f"[Running KASE {kase}]")
+            for testcase in self.testcases:
+                self.run_single_test(testcase=testcase, kase=kase)
+

scripts/tester.py

@@ -0,0 +1,126 @@
+import os
+import subprocess
+from analyzer import Analyzer
+from compiler import Compiler
+from runner import Runner
+from pretty_print import Print_C
+from presenter import Presenter
+
+
+def get_sy_testcases(sub_dir=""):
+    filelist = [] #not_file_list = ["063_sort_test3", "10_break", "55_sort_test3", "62_long_code"]
+    not_file_list = []
+    for file in os.listdir(os.path.join("testcases", sub_dir)):
+        if os.path.splitext(file)[1] == ".sy" and not(os.path.splitext(file)[0] in not_file_list):
+            filelist.append(os.path.splitext(file)[0])
+    return sorted(filelist)
+
+class Tester:
+    def __init__(self, a_scheme, is_trivial, testcases):
+        self.scheme = a_scheme["scheme"]
+        self.frontend_instr = a_scheme["frontend_instr"]
+        self.emit_llvm_ir = a_scheme["emit_llvm_ir"]
+        self.is_trivial = is_trivial
+
+        self.compiler = Compiler(scheme=self.scheme, testcases=testcases)
+        self.runner = Runner(scheme=self.scheme, testcases=testcases)
+        self.analyzer = Analyzer(scheme=self.scheme, emit_llvm_ir=self.emit_llvm_ir, testcases=testcases)
+
+
+    def generate_ir(self):
+        for testcase in testcases: self.compiler.sy_to_ir(frontend_instr=self.frontend_instr, testcase=testcase)
+
+
+    def compile(self):
+        self.compiler.compile_all_tests(frontend_instr=self.frontend_instr, emit_llvm_ir=self.emit_llvm_ir)
+
+    def compile_rubbish(self):
+        self.compiler.compile_rubbish(frontend_instr=self.frontend_instr, category="functional")
+
+    def run(self):
+        self.runner.run_all_tests()
+
+
+    def analyze(self):
+        self.analyzer.analyze()
+
+
+    def test(self):
+        Print_C.print_header(f"[TESTING {self.scheme}]")
+        self.compile()
+        self.run()
+        self.analyze()
+        print()
+        print()
+
+
+    def test_ir(self):
+        Print_C.print_header(f"[TESTING {self.scheme}] (IR only)")
+        for testcase in testcases:
+            Print_C.print_subheader(f"[Compiling {testcase} with {self.scheme}]")
+            self.compiler.sy_to_ir(frontend_instr=self.frontend_instr, testcase=testcase)
+        print()
+        print()
+
+thu_compiler = "build/bin/thu_compiler "
+ustc_compiler = "build/bin/ustc_compiler "
+ustc_compiler_no_vec = "build/bin/ustc_compiler_no_vec "
+
+clang_llvm_scheme = {"scheme": "clang_llvm",
+                     "frontend_instr": "clang -x c -c -Ofast -mcpu=cortex-a72 -mfpu=neon -mfloat-abi=hard -S -emit-llvm -include {header} {sy} -o {ir}",
+                     "emit_llvm_ir": True}
+
+thu_llvm_scheme = {"scheme": "thu_llvm",
+                   "frontend_instr": thu_compiler + "-l {ir} {sy}",
+                   "emit_llvm_ir": True}
+
+thu_thu_scheme = {"scheme": "thu_thu",
+                  "frontend_instr": thu_compiler + "-o {asm} {sy}",
+                  "emit_llvm_ir": False}
+
+my_compiler = "build/sysy"
+
+my_scheme = {"scheme": "my",
+              "frontend_instr": my_compiler + " -S {sy} -o {ir}",
+              "emit_llvm_ir": False}
+
+# ustc_ustc_scheme = {"scheme": "ustc_ustc",
+#                     "frontend_instr": ustc_compiler + "-o {asm} {sy}",
+#                     "emit_llvm_ir": False}
+
+# ustc_ustc_no_vec_scheme = {
+#                     "scheme": "ustc_ustc_no_vec",
+#                     "frontend_instr": ustc_compiler_no_vec + "-o {asm} {sy}",
+#                     "emit_llvm_ir": False}
+
+# gcc_gcc_scheme = {"scheme": "gcc_gcc",
+#                   "frontend_instr": "gcc -x c -c -Ofast -mcpu=cortex-a72 -mfpu=neon -mfloat-abi=hard -S -include {header} {sy} -o {asm}",
+#                   "emit_llvm_ir": False}
+
+# ustc_llvm_scheme = {"scheme": "ustc_llvm", # generate ir only
+#                     "frontend_instr": ustc_compiler + "-emit -o {ir} {sy}",
+#                     "emit_llvm_ir": True}
+
+
+if __name__ == '__main__':
+    testcases = get_sy_testcases(sub_dir="functional")
+    all_schemes = [clang_llvm_scheme, thu_llvm_scheme, thu_thu_scheme] # gcc_gcc_scheme, ustc_ustc_scheme, ustc_ustc_no_vec_scheme]
+    testers = []
+    Print_C.print_header("[Removing old data...]\n\n")
+    subprocess.run("rm -rf build/test_results/".split())
+    subprocess.run("rm -rf build/output/".split())
+    subprocess.run("rm -rf build/log/compile_log".split())
+    subprocess.run("rm -rf build/log/run_log".split())
+    subprocess.run("rm -rf build/log/test_result.log".split())
+
+    Tester(my_scheme, True, testcases).compile_rubbish()
+
+    # for scheme in all_schemes:
+    #     tester = Tester(scheme, is_trivial=True)
+    #     testers.append(tester)
+    #     tester.test()
+
+    # # Tester(ustc_llvm_scheme).test_ir()
+
+    # presenter = Presenter(schemes=[scheme["scheme"] for scheme in all_schemes], testcases=get_sy_testcases())
+    # presenter.present_all_testcases()