diff --git a/.gitignore b/.gitignore index 9c3e147..3c73ba5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ build/ .antlr/ -*.log \ No newline at end of file +*.log +__pycache__/ \ No newline at end of file diff --git a/include/common.h b/include/common.h index 98043df..e899b5e 100644 --- a/include/common.h +++ b/include/common.h @@ -15,7 +15,7 @@ #define sysy_assert(cond) \ do { \ if (!(cond)) \ - throw GrammarException(__FILE__, __LINE__, #cond); \ + throw GrammarException(__FILE__, __LINE__, #cond); \ } while (0) namespace antlrSysY { diff --git a/scripts/analyzer.py b/scripts/analyzer.py new file mode 100644 index 0000000..6efcb61 --- /dev/null +++ b/scripts/analyzer.py @@ -0,0 +1,47 @@ +import os +import subprocess +from pretty_print import Print_C + +class Analyzer: + def __init__(self, scheme, emit_llvm_ir, testcases) -> None: + self.scheme = scheme + self.emit_llvm_ir = emit_llvm_ir + self.testcases = testcases + + self.asm_template = f"build/test_results/{{testcase}}/asm/{scheme}.s" + self.perf_template = f"build/test_results/{{testcase}}/analyze/perf/{scheme}.txt" + self.llvm_mca_template = f"build/test_results/{{testcase}}/analyze/llvm_mca/{scheme}.txt" + + for testcase in testcases: + self.__generate_path(testcase) + + def __generate_path(self, testcase): + llvm_mca_path = f"build/test_results/{testcase}/analyze/llvm_mca/" + perf_path = f"build/test_results/{testcase}/analyze/perf/" + + if not os.path.exists(llvm_mca_path): + os.makedirs(llvm_mca_path) + + if not os.path.exists(perf_path): + os.makedirs(perf_path) + + def run_perf(self): # TODO + pass + + def run_llvm_mca(self, testcase): + asm = self.asm_template.format(testcase=testcase) + llvm_mca = self.llvm_mca_template.format(testcase=testcase) + + llvm_mca_file = open(llvm_mca, "w+") + + subprocess.run(f"llvm-mca -mtriple=armv7 -mcpu=cortex-a72 -timeline {asm}".split(), stdout=llvm_mca_file, bufsize=1) + + llvm_mca_file.close() + + + def analyze(self): + for testcase in self.testcases: + Print_C.print_subheader(f"[Analyzing {self.scheme} | {testcase}]") + # self.run_llvm_mca(testcase=testcase) + + # self.run_perf() \ No newline at end of file diff --git a/scripts/compiler.py b/scripts/compiler.py new file mode 100644 index 0000000..8fa21dd --- /dev/null +++ b/scripts/compiler.py @@ -0,0 +1,146 @@ +import os +import subprocess +from pretty_print import Print_C + +lib = "build/lib/libsysy.a" +header = "build/include/sylib.h" + +class Compiler: + def __init__(self, scheme, testcases): + self.scheme = scheme + self.testcases = testcases + + self.sy_template = f"testcases/{{testcase}}.sy" + self.sy_fix_template = f"testcases/{{testcase}}.{scheme}_sy" + self.sy_category_template = f"testcases/{{category}}/{{testcase}}.sy" + self.llvm_ir_template = f"build/test_results/{{testcase}}/llvm_ir/{scheme}.ll" + self.asm_template = f"build/test_results/{{testcase}}/asm/{scheme}.s" + self.obj_template = f"build/test_results/{{testcase}}/obj/{scheme}.o" + self.bin_template = f"build/test_results/{{testcase}}/bin/{scheme}" + self.compile_log = f"build/log/compile_log/{{testcase}}/{scheme}.log" + + for testcase in testcases: + self.__generate_path(testcase) + + + def __generate_path(self, testcase): + llvm_ir_path = f"build/test_results/{testcase}/llvm_ir/" + asm_path = f"build/test_results/{testcase}/asm/" + obj_path = f"build/test_results/{testcase}/obj/" + bin_path = f"build/test_results/{testcase}/bin/" + compile_log_path = f"build/log/compile_log/{testcase}/" + + if not os.path.exists(llvm_ir_path): + os.makedirs(llvm_ir_path) + + if not os.path.exists(asm_path): + os.makedirs(asm_path) + + if not os.path.exists(obj_path): + os.makedirs(obj_path) + + if not os.path.exists(bin_path): + os.makedirs(bin_path) + + if not os.path.exists(compile_log_path): + os.makedirs(compile_log_path) + + + def sy_to_ir(self, frontend_instr, testcase, category=""): + sy = self.sy_template.format(testcase=testcase) + sy_fix = self.sy_fix_template.format(testcase=testcase) + sy_category = self.sy_category_template.format(testcase=testcase, category=category) + if os.path.exists(sy_fix): + sy = sy_fix + elif os.path.exists(sy_category): + sy = sy_category + + ir = self.llvm_ir_template.format(testcase=testcase) + log = self.compile_log.format(testcase=testcase) + + log_file = open(log, "a+") + + Print_C.print_procedure(f"Generating {self.scheme}.ll") + completed = subprocess.run(frontend_instr.format(header=header, sy=sy, ir=ir).split(), stdout=log_file, stderr=log_file, bufsize=1) + if completed.returncode != 0: + Print_C.print_error(f"Generating {self.scheme}.ll failed! See {log}") + log_file.close() + + + def ir_to_asm(self, testcase): + ir = self.llvm_ir_template.format(testcase=testcase) + asm = self.asm_template.format(testcase=testcase) + log = self.compile_log.format(testcase=testcase) + + log_file = open(log, "a+") + + Print_C.print_procedure(f"Generating {self.scheme}.s") + subprocess.run(f"llc -O3 -march=arm -mcpu=cortex-a72 -float-abi=hard -filetype=asm {ir} -o {asm}".split(), stdout=log_file, stderr=log_file, bufsize=1) + + log_file.close() + + + def asm_to_obj(self, testcase): + asm = self.asm_template.format(testcase=testcase) + obj = self.obj_template.format(testcase=testcase) + log = self.compile_log.format(testcase=testcase) + + log_file = open(log, "a+") + + Print_C.print_procedure(f"Generating {self.scheme}.o") + subprocess.run(f"as -march=armv7-a -mfloat-abi=hard {asm} -o {obj}".split(), stdout=log_file, stderr=log_file, bufsize=1) + + log_file.close() + + def obj_to_bin(self, testcase): + obj = self.obj_template.format(testcase=testcase) + bin = self.bin_template.format(testcase=testcase) + log = self.compile_log.format(testcase=testcase) + + log_file = open(log, "a+") + + Print_C.print_procedure(f"Generating {self.scheme}") + subprocess.run(f"clang -Ofast -marm -march=armv7-a -mfpu=neon -mfloat-abi=hard {obj} {lib} -o {bin}".split(), stdout=log_file, stderr=log_file, bufsize=1) + + log_file.close() + + + def sy_to_asm(self, frontend_instr, testcase): + asm = self.asm_template.format(testcase=testcase) + sy = self.sy_template.format(testcase=testcase) + sy_fix = self.sy_fix_template.format(testcase=testcase) + if os.path.exists(sy_fix): + sy = sy_fix + + log = self.compile_log.format(testcase=testcase) + + log_file = open(log, "a+") + + Print_C.print_procedure(f"Generating {self.scheme}.s") + subprocess.run(frontend_instr.format(header=header, asm=asm, sy=sy).split(), stdout=log_file, stderr=log_file, bufsize=1) + + + log_file.close() + + + def compile_all_tests(self, frontend_instr, emit_llvm_ir): + if emit_llvm_ir: + for testcase in self.testcases: + Print_C.print_subheader(f"[Compiling {self.scheme} | {testcase}]") + self.sy_to_ir(frontend_instr=frontend_instr, testcase=testcase) + self.ir_to_asm(testcase=testcase) + self.asm_to_obj(testcase=testcase) + self.obj_to_bin(testcase=testcase) + else: + for testcase in self.testcases: + Print_C.print_subheader(f"[Compiling {self.scheme} | {testcase}]") + self.sy_to_asm(frontend_instr=frontend_instr, testcase=testcase) + self.asm_to_obj(testcase=testcase) + self.obj_to_bin(testcase=testcase) + + def compile_rubbish(self, frontend_instr, category): + for testcase in self.testcases: + Print_C.print_subheader(f"[Compiling {self.scheme} | {testcase}]") + self.sy_to_ir(frontend_instr=frontend_instr, testcase=testcase, category=category) + + \ No newline at end of file diff --git a/scripts/presenter.py b/scripts/presenter.py new file mode 100644 index 0000000..0032a41 --- /dev/null +++ b/scripts/presenter.py @@ -0,0 +1,87 @@ +import re +from runner import Runner +from collections import Counter +from pretty_print import Print_C + +class Presenter: + kases = Runner.run_kases + splitter = re.compile('[-@ :\t\r\n]+') + scheme_time_print_template = "{0:<10} : {1:<10.0f}us" + + def __init__(self, schemes, testcases): + self.schemes = schemes + self.testcases = testcases + + def __diff(self, file1, file2): + file1_lines = open(file1).readlines() + file2_lines = open(file2).readlines() + return file1_lines == file2_lines + + def __time_to_us(self, l): + hours = int(l[0][:-1]) + minutes = int(l[1][:-1]) + seconds = int(l[2][:-1]) + us = int(l[3][:-2]) + return hours * 60 * 60 * 1000000 + minutes * 60 * 1000000 + seconds * 1000000 + us + + def present_single_testcase(self, testcase): + Print_C.print_subheader(f"[Checking {testcase}]") + + scheme_time_counter = Counter({scheme: 0 for scheme in self.schemes}) # {scheme: time} + stage_scheme_time_dict = {} + wrong_schemes = [] + for scheme in self.schemes: + stdout = f"testcases/{testcase}.out" + myout = f"build/output/{testcase}/{scheme}.out" + if not self.__diff(stdout, myout): + wrong_schemes.append(scheme) + continue + + stage_time_counter = Counter({}) # {stage: time} + for kase in range(Presenter.kases): + runner_log = f"build/log/run_log/{testcase}/{scheme}_{kase}.out" + lines = open(runner_log).readlines() + # ['Timer', 'startLine', 'endLine', 'xH', 'xM', 'xS', 'xus'][1:] + parsed_lines = [Presenter.splitter.split(line)[1:] for line in lines][:-1] + stage_time = {f"{int(line[0])} ~ {int(line[1])}": self.__time_to_us(line[2:]) for line in parsed_lines} + stage_time_counter += Counter(stage_time) + + for stage_time in stage_time_counter.items(): + if not stage_time[0] in stage_scheme_time_dict: + stage_scheme_time_dict[stage_time[0]] = {} + stage_scheme_time_dict[stage_time[0]][scheme] = stage_time[1] / Presenter.kases + + scheme_time_counter[scheme] = sum(stage_time_counter.values()) / Presenter.kases + + Print_C.print_procedure("[AVERAGE TIME COUNT (sorted)]") + sorted_scheme_time_list = sorted(scheme_time_counter.items(), key=lambda scheme_time: scheme_time[1]) + for scheme_time in sorted_scheme_time_list: + if not scheme_time[0] in wrong_schemes: + Print_C.print_pass(self.scheme_time_print_template.format(scheme_time[0], scheme_time[1])) + for scheme in wrong_schemes: + Print_C.print_error(f"{scheme} WA") + + for stage_scheme_time in stage_scheme_time_dict.items(): + Print_C.print_procedure(f"[Line {stage_scheme_time[0]}]") + + sorted_stage_scheme_list = sorted(stage_scheme_time[1].items(), key=lambda scheme_time: scheme_time[1]) + for scheme_time in sorted_stage_scheme_list: + print(self.scheme_time_print_template.format(scheme_time[0], scheme_time[1])) + + return scheme_time_counter + + def present_all_testcases(self): + Print_C.print_header(f"[Checking & Racing]") + scheme_tot_time_counter = Counter({scheme: 0 for scheme in self.schemes}) # {scheme: total_time} + + for testcase in self.testcases: + scheme_tot_time_counter += Counter(self.present_single_testcase(testcase=testcase)) + print() + + print() + sorted_scheme_tot_time_list = sorted(scheme_tot_time_counter.items(), key=lambda scheme_tot_time: scheme_tot_time[1]) + Print_C.print_header(f"[TOTAL TIME RANKING]") + for scheme_tot_time in sorted_scheme_tot_time_list: + print(self.scheme_time_print_template.format(scheme_tot_time[0], scheme_tot_time[1])) + print() + print() \ No newline at end of file diff --git a/scripts/pretty_print.py b/scripts/pretty_print.py new file mode 100644 index 0000000..3f56444 --- /dev/null +++ b/scripts/pretty_print.py @@ -0,0 +1,40 @@ +import os +class Print_C: + GREEN = '\033[32m' + RED = '\033[31m' + YELLOW = '\033[33m' + BLUE = '\033[34m' + BOLD = '\033[1m' + ENDC = '\033[0m' + if not os.path.exists("build/log"): + os.makedirs("build/log") + log = "build/log/test_result.log" + log_file = open(log, "a+") + + def print_error(msg): + content = f"{Print_C.RED}{msg}{Print_C.ENDC}" + print(content) + Print_C.log_file.write(content + "\n") + + + def print_header(msg): + content = f"{Print_C.GREEN}{Print_C.BOLD}{msg}{Print_C.ENDC}" + print(content) + Print_C.log_file.write(content + "\n") + + def print_pass(msg): + content = f"{Print_C.GREEN}{msg}{Print_C.ENDC}" + print(content) + Print_C.log_file.write(content + "\n") + + + def print_subheader(msg): + content = f"{Print_C.YELLOW}{msg}{Print_C.ENDC}" + print(content) + Print_C.log_file.write(content + "\n") + + + def print_procedure(msg): + content = f"{Print_C.BLUE}{msg}{Print_C.ENDC}" + print(content) + Print_C.log_file.write(content + "\n") \ No newline at end of file diff --git a/scripts/runner.py b/scripts/runner.py new file mode 100644 index 0000000..32ed59c --- /dev/null +++ b/scripts/runner.py @@ -0,0 +1,69 @@ +import os +import subprocess +from pretty_print import Print_C + +class Runner: + run_kases = 3 + + def __init__(self, scheme, testcases): + self.scheme = scheme + self.testcases = testcases + + self.bin_file_template = f"build/test_results/{{testcase}}/bin/{scheme}" + self.myout_template = f"build/output/{{testcase}}/{scheme}.out" + self.runner_log = f"build/log/run_log/{{testcase}}/{scheme}_{{kase}}.out" + + for testcase in testcases: + self.__generate_path(testcase) + + + def __generate_path(self, testcase): + myout_path = f"build/output/{testcase}/" + runner_log_path = f"build/log/run_log/{testcase}/" + + if not os.path.exists(myout_path): + os.makedirs(myout_path) + + if not os.path.exists(runner_log_path): + os.makedirs(runner_log_path) + + + def run_single_test(self, testcase, kase): + bin = self.bin_file_template.format(testcase=testcase) + stdin = f"testcases/{testcase}.in" + myout = self.myout_template.format(testcase=testcase) + log = self.runner_log.format(testcase=testcase, kase=kase) + + myout_file = open(myout, "a+") + log_file = open(log, "a+") + null_file = open(os.devnull, "w") + + Print_C.print_procedure(f"Running {self.scheme}_{testcase} [kase: {kase}]") + + if os.path.exists(stdin): + stdin_file = open(stdin, "r") + if kase == 0: + p = subprocess.run(f"{bin}".split(), stdin=stdin_file, stdout=myout_file, stderr=log_file, bufsize=1) + subprocess.run(f"echo".split(), stdout=myout_file, bufsize=1) + subprocess.run(f"echo {p.returncode}".split(), stdout=myout_file, bufsize=1) + else: + p = subprocess.run(f"{bin}".split(), stdin=stdin_file, stdout=null_file, stderr=null_file, bufsize=1) + stdin_file.close() + else: + if kase == 0: + p = subprocess.run(f"{bin}".split(), stdout=myout_file, stderr=log_file, bufsize=1) + subprocess.run(f"echo".split(), stdout=myout_file, bufsize=1) + subprocess.run(f"echo {p.returncode}".split(), stdout=myout_file, bufsize=1) + else: + p = subprocess.run(f"{bin}".split(), stdout=null_file, stderr=null_file, bufsize=1) + + myout_file.close() + log_file.close() + + + def run_all_tests(self): + for kase in range(Runner.run_kases): + Print_C.print_subheader(f"[Running KASE {kase}]") + for testcase in self.testcases: + self.run_single_test(testcase=testcase, kase=kase) + diff --git a/scripts/tester.py b/scripts/tester.py new file mode 100644 index 0000000..b0684fe --- /dev/null +++ b/scripts/tester.py @@ -0,0 +1,126 @@ +import os +import subprocess +from analyzer import Analyzer +from compiler import Compiler +from runner import Runner +from pretty_print import Print_C +from presenter import Presenter + + +def get_sy_testcases(sub_dir=""): + filelist = [] #not_file_list = ["063_sort_test3", "10_break", "55_sort_test3", "62_long_code"] + not_file_list = [] + for file in os.listdir(os.path.join("testcases", sub_dir)): + if os.path.splitext(file)[1] == ".sy" and not(os.path.splitext(file)[0] in not_file_list): + filelist.append(os.path.splitext(file)[0]) + return sorted(filelist) + +class Tester: + def __init__(self, a_scheme, is_trivial, testcases): + self.scheme = a_scheme["scheme"] + self.frontend_instr = a_scheme["frontend_instr"] + self.emit_llvm_ir = a_scheme["emit_llvm_ir"] + self.is_trivial = is_trivial + + self.compiler = Compiler(scheme=self.scheme, testcases=testcases) + self.runner = Runner(scheme=self.scheme, testcases=testcases) + self.analyzer = Analyzer(scheme=self.scheme, emit_llvm_ir=self.emit_llvm_ir, testcases=testcases) + + + def generate_ir(self): + for testcase in testcases: self.compiler.sy_to_ir(frontend_instr=self.frontend_instr, testcase=testcase) + + + def compile(self): + self.compiler.compile_all_tests(frontend_instr=self.frontend_instr, emit_llvm_ir=self.emit_llvm_ir) + + def compile_rubbish(self): + self.compiler.compile_rubbish(frontend_instr=self.frontend_instr, category="functional") + + def run(self): + self.runner.run_all_tests() + + + def analyze(self): + self.analyzer.analyze() + + + def test(self): + Print_C.print_header(f"[TESTING {self.scheme}]") + self.compile() + self.run() + self.analyze() + print() + print() + + + def test_ir(self): + Print_C.print_header(f"[TESTING {self.scheme}] (IR only)") + for testcase in testcases: + Print_C.print_subheader(f"[Compiling {testcase} with {self.scheme}]") + self.compiler.sy_to_ir(frontend_instr=self.frontend_instr, testcase=testcase) + print() + print() + +thu_compiler = "build/bin/thu_compiler " +ustc_compiler = "build/bin/ustc_compiler " +ustc_compiler_no_vec = "build/bin/ustc_compiler_no_vec " + +clang_llvm_scheme = {"scheme": "clang_llvm", + "frontend_instr": "clang -x c -c -Ofast -mcpu=cortex-a72 -mfpu=neon -mfloat-abi=hard -S -emit-llvm -include {header} {sy} -o {ir}", + "emit_llvm_ir": True} + +thu_llvm_scheme = {"scheme": "thu_llvm", + "frontend_instr": thu_compiler + "-l {ir} {sy}", + "emit_llvm_ir": True} + +thu_thu_scheme = {"scheme": "thu_thu", + "frontend_instr": thu_compiler + "-o {asm} {sy}", + "emit_llvm_ir": False} + +my_compiler = "build/sysy" + +my_scheme = {"scheme": "my", + "frontend_instr": my_compiler + " -S {sy} -o {ir}", + "emit_llvm_ir": False} + +# ustc_ustc_scheme = {"scheme": "ustc_ustc", +# "frontend_instr": ustc_compiler + "-o {asm} {sy}", +# "emit_llvm_ir": False} + +# ustc_ustc_no_vec_scheme = { +# "scheme": "ustc_ustc_no_vec", +# "frontend_instr": ustc_compiler_no_vec + "-o {asm} {sy}", +# "emit_llvm_ir": False} + +# gcc_gcc_scheme = {"scheme": "gcc_gcc", +# "frontend_instr": "gcc -x c -c -Ofast -mcpu=cortex-a72 -mfpu=neon -mfloat-abi=hard -S -include {header} {sy} -o {asm}", +# "emit_llvm_ir": False} + +# ustc_llvm_scheme = {"scheme": "ustc_llvm", # generate ir only +# "frontend_instr": ustc_compiler + "-emit -o {ir} {sy}", +# "emit_llvm_ir": True} + + +if __name__ == '__main__': + testcases = get_sy_testcases(sub_dir="functional") + all_schemes = [clang_llvm_scheme, thu_llvm_scheme, thu_thu_scheme] # gcc_gcc_scheme, ustc_ustc_scheme, ustc_ustc_no_vec_scheme] + testers = [] + Print_C.print_header("[Removing old data...]\n\n") + subprocess.run("rm -rf build/test_results/".split()) + subprocess.run("rm -rf build/output/".split()) + subprocess.run("rm -rf build/log/compile_log".split()) + subprocess.run("rm -rf build/log/run_log".split()) + subprocess.run("rm -rf build/log/test_result.log".split()) + + Tester(my_scheme, True, testcases).compile_rubbish() + + # for scheme in all_schemes: + # tester = Tester(scheme, is_trivial=True) + # testers.append(tester) + # tester.test() + + # # Tester(ustc_llvm_scheme).test_ir() + + # presenter = Presenter(schemes=[scheme["scheme"] for scheme in all_schemes], testcases=get_sy_testcases()) + # presenter.present_all_testcases()