diff --git a/.github/workflows/vfc_test_workflow.yml b/.github/workflows/vfc_test_workflow.yml new file mode 100644 index 0000000..61bad51 --- /dev/null +++ b/.github/workflows/vfc_test_workflow.yml @@ -0,0 +1,57 @@ +# This workflow will be executed when dev is updated: +# it will run the configured tests and upload the results on vfc_ci_dev. + +name: "Verificarlo CI (dev)" + +on: + # Triggers the workflow when dev is updated + push: + branches: [ dev ] + + workflow_dispatch: + + +jobs: + run_verificarlo_tests: + runs-on: ubuntu-latest + container: verificarlo/verificarlo + + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + + # We will probably drop these installations when integrating CI into + # Verificarlo + - name: Install Python requirements + run: | + pip install numpy scipy pandas bokeh jinja2 tables GitPython + apt update + apt install -y wget libhdf5-dev g++ + wget https://raw.githubusercontent.com/verificarlo/significantdigits/main/sigdigits.py -P /usr/local/lib/python3.8/dist-packages + + - name: Run tests + # We assume the script is included in the repo for now + # (we'll probably want to remove "./" if the script ends up being integrated + # in Verificarlo and becomes available system-wide) + run: ./vfc_ci test -g -r + + - name: Commit test results + run: | + git_hash=$(git rev-parse --short "$GITHUB_SHA") + + git config --local user.email "action@github.com" + git config --local user.name "GitHub Action" + + git checkout vfc_ci_dev + mkdir -p vfcruns + mv *.vfcrun.h5 vfcruns + git add vfcruns/* + git commit -m "[auto] New test results for commit ${git_hash}" + git push + + - name: Upload raw results as artifacts + uses: actions/upload-artifact@v2 + with: + name: ${{github.sha}}.vfcraw + path: ./*.vfcraw.h5 diff --git a/.gitignore b/.gitignore index 9980d03..f372432 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,8 @@ Slater* Updates* datasets/dataset.* bin/ +*.vfcrun.hd5 +*.vfcraw.hd5 +__pycache__ +ci/__pycache__ +*.h5 diff --git a/Makefile.verificarlo b/Makefile.verificarlo index 167b4b2..bd10702 100644 --- a/Makefile.verificarlo +++ b/Makefile.verificarlo @@ -33,6 +33,7 @@ BIN_DIR := bin EXEC := $(BIN_DIR)/cMaponiA3_test_3x3_3 \ $(BIN_DIR)/test_h5 \ + $(BIN_DIR)/vfc_test_h5 \ $(BIN_DIR)/fMaponiA3_test_3x3_3 \ $(BIN_DIR)/fMaponiA3_test_4x4_2 \ $(BIN_DIR)/QMCChem_dataset_test @@ -84,6 +85,9 @@ $(BIN_DIR)/cMaponiA3_test_3x3_3: $(OBJ_DIR)/cMaponiA3_test_3x3_3.o $(DEPS_CXX) | $(BIN_DIR)/test_h5: $(OBJ_DIR)/test_h5.o $(DEPS_CXX) | $(BIN_DIR) $(CXX) -o $@ $^ $(H5FLAGS) +$(BIN_DIR)/vfc_test_h5: $(OBJ_DIR)/vfc_test_h5.o $(DEPS_CXX) | $(BIN_DIR) + $(CXX) -o $@ $^ $(H5FLAGS) + $(BIN_DIR)/fMaponiA3_test_3x3_3: $(DEPS_F) $(OBJ_DIR)/fMaponiA3_test_3x3_3.o | $(BIN_DIR) $(FC) $(FLIBS) -o $@ $^ @@ -92,5 +96,3 @@ $(BIN_DIR)/fMaponiA3_test_4x4_2: $(DEPS_F) $(OBJ_DIR)/fMaponiA3_test_4x4_2.o | $(BIN_DIR)/QMCChem_dataset_test: $(DEPS_F) $(OBJ_DIR)/QMCChem_dataset_test.o | $(BIN_DIR) $(FC) $(FLIBS) -o $@ $^ - - diff --git a/ci/__init__.py b/ci/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ci/serve.py b/ci/serve.py new file mode 100755 index 0000000..37d91f2 --- /dev/null +++ b/ci/serve.py @@ -0,0 +1,28 @@ +# Server for the Verificarlo CI report. This is simply a wrapper to avoid +# calling Bokeh directly. + +import os + + +def serve(show, git_directory, git_url, port, allow_origin, logo_url): + + # Prepare arguments + show = "--show" if show else "" + + git = "" + if git_directory is not None: + git = "git directory %s" % git_directory + if git_url is not None: + git = "git url %s" % git_url + + logo = "" + if logo_url is not None: + logo = "logo %s" % logo_url + + dirname = os.path.dirname(__file__) + + # Call the "bokeh serve" command on the system + command = "bokeh serve %s/vfc_ci_report %s --allow-websocket-origin=%s:%s --port %s --args %s %s" \ + % (dirname, show, allow_origin, port, port, git, logo) + + os.system(command) diff --git a/ci/setup.py b/ci/setup.py new file mode 100644 index 0000000..1bd50b3 --- /dev/null +++ b/ci/setup.py @@ -0,0 +1,146 @@ +# Helper script to set up Verificarlo CI on the current branch + +import git + +import sys +import os +from jinja2 import Environment, FileSystemLoader + +########################################################################## + +# Helper functions + + +def gen_readme(dev_branch, ci_branch): + + # Init template loader + path = os.path.dirname(os.path.abspath(__file__)) + env = Environment(loader=FileSystemLoader(path)) + template = env.get_template("workflow_templates/ci_README.j2.md") + + # Render template + render = template.render(dev_branch=dev_branch, ci_branch=ci_branch) + + # Write template + with open("README.md", "w") as fh: + fh.write(render) + + +def gen_workflow(git_host, dev_branch, ci_branch, repo): + + # Init template loader + path = os.path.dirname(os.path.abspath(__file__)) + env = Environment(loader=FileSystemLoader(path)) + + if git_host == "github": + # Load template + template = env.get_template( + "workflow_templates/vfc_test_workflow.j2.yml") + + # Render it + render = template.render(dev_branch=dev_branch, ci_branch=ci_branch) + + # Write the file + filename = ".github/workflows/vfc_test_workflow.yml" + os.makedirs(os.path.dirname(filename), exist_ok=True) + with open(filename, "w") as fh: + fh.write(render) + + if git_host == "gitlab": + template = env.get_template("workflow_templates/gitlab-ci.j2.yml") + + # Ask for the user who will run the jobs (Gitlab specific) + username = input( + "[vfc_ci] Enter the name of the user who will run the CI jobs:") + email = input( + "[vfc_ci] Enter the e-mail of the user who will run the CI jobs:") + + remote_url = repo.remotes[0].config_reader.get("url") + remote_url = remote_url.replace("http://", "") + remote_url = remote_url.replace("https://", "") + + render = template.render( + dev_branch=dev_branch, + ci_branch=ci_branch, + username=username, + email=email, + remote_url=remote_url + ) + + filename = ".gitlab-ci.yml" + with open(filename, "w") as fh: + fh.write(render) + + +########################################################################## + +def setup(git_host): + + # Init repo and make sure that the workflow setup is possible + + repo = git.Repo(".") + repo.remotes.origin.fetch() + + # Make sure that repository is clean + assert(not repo.is_dirty()), "Error [vfc_ci]: Unstaged changes detected " \ + "in your work tree." + + dev_branch = repo.active_branch + dev_branch_name = str(dev_branch) + dev_remote = dev_branch.tracking_branch() + + # Make sure that the active branch (on which to setup the workflow) has a + # remote + assert(dev_remote is not None), "Error [vfc_ci]: The current branch doesn't " \ + "have a remote." + + # Make sure that we are not behind the remote (so we can push safely later) + rev = "%s...%s" % (dev_branch_name, str(dev_remote)) + commits_behind = list(repo.iter_commits(rev)) + assert(commits_behind == []), "Error [vfc_ci]: The local branch seems " \ + "to be at least one commit behind remote." + + # Commit the workflow on the current (dev) branch + + ci_branch_name = "vfc_ci_%s" % dev_branch_name + gen_workflow(git_host, dev_branch_name, ci_branch_name, repo) + repo.git.add(".") + repo.index.commit("[auto] Set up Verificarlo CI on this branch") + repo.remote(name="origin").push() + + # Create the CI branch (orphan branch with a readme on it) + # (see : https://github.com/gitpython-developers/GitPython/issues/615) + + repo.head.reference = git.Head(repo, "refs/heads/" + ci_branch_name) + + repo.index.remove(["*"]) + gen_readme(dev_branch_name, ci_branch_name) + repo.index.add(["README.md"]) + + repo.index.commit( + "[auto] Create the Verificarlo CI branch for %s" % dev_branch_name, + parent_commits=None + ) + repo.remote(name="origin").push( + refspec="%s:%s" % (ci_branch_name, ci_branch_name) + ) + + # Force checkout back to the original (dev) branch + repo.git.checkout(dev_branch_name, force=True) + + # Print termination messages + + print( + "Info [vfc_ci]: A Verificarlo CI workflow has been setup on " + "%s." % dev_branch_name + ) + print( + "Info [vfc_ci]: Make sure that you have a \"vfc_tests_config.json\" on " + "this branch. You can also perform a \"vfc_ci test\" dry run before " + "pushing other commits.") + + if git_host == "gitlab": + print( + "Info [vfc_ci]: Since you are using GitLab, make sure that you " + "have created an access token for the user you specified (registered " + "as a variable called \"CI_PUSH_TOKEN\" in your repository).") diff --git a/ci/test.py b/ci/test.py new file mode 100755 index 0000000..bc38714 --- /dev/null +++ b/ci/test.py @@ -0,0 +1,367 @@ +# This script reads the vfc_tests_config.json file and executes tests accordingly +# It will also generate a ... .vfcrunh5 file with the results of the run + +import sigdigits as sd +import scipy.stats +import numpy as np +import pandas as pd +import os +import json + +import calendar +import time + +# Forcing an older pickle protocol allows backwards compatibility when reading +# HDF5 written in 3.8+ using an older version of Python +import pickle +pickle.HIGHEST_PROTOCOL = 4 + + +# Magic numbers +min_pvalue = 0.05 +max_zscore = 3 + + +########################################################################## + +# Helper functions + +# Read a CSV file outputted by vfc_probe as a Pandas dataframe +def read_probes_csv(filepath, backend, warnings, execution_data): + + try: + results = pd.read_csv(filepath) + + except FileNotFoundError: + print( + "Warning [vfc_ci]: Probes not found, your code might have crashed " + "or you might have forgotten to call vfc_dump_probes" + ) + warnings.append(execution_data) + return pd.DataFrame( + columns=["test", "variable", "values", "vfc_backend"] + ) + + except Exception: + print( + "Warning [vfc_ci]: Your probes could not be read for some unknown " + "reason" + ) + warnings.append(execution_data) + return pd.DataFrame( + columns=["test", "variable", "values", "vfc_backend"] + ) + + if len(results) == 0: + print( + "Warning [vfc_ci]: Probes empty, it looks like you have dumped " + "them without calling vfc_put_probe" + ) + warnings.append(execution_data) + + # Once the CSV has been opened and validated, return its content + results["value"] = results["value"].apply(lambda x: float.fromhex(x)) + results.rename(columns={"value": "values"}, inplace=True) + + results["vfc_backend"] = backend + + return results + + +# Wrappers to sd.significant_digits (returns results in base 2) + +def significant_digits(x): + + # If the null hypothesis is rejected, call sigdigits with the General + # formula: + if x.pvalue < min_pvalue: + # In a pandas DF, "values" actually refers to the array of columns, and + # not the column named "values" + distribution = x.values[3] + distribution = distribution.reshape(len(distribution), 1) + + # The distribution's empirical average will be used as the reference + mu = np.array([x.mu]) + + s = sd.significant_digits( + distribution, + mu, + precision=sd.Precision.Relative, + method=sd.Method.General, + + probability=0.9, + confidence=0.95 + ) + + # s is returned inside a list + return s[0] + + # Else, manually compute sMCA (Stott-Parker formula) + else: + return -np.log2(np.absolute(x.sigma / x.mu)) + + +def significant_digits_lower_bound(x): + # If the null hypothesis is rejected, no lower bound + if x.pvalue < min_pvalue: + return x.s2 + + # Else, the lower bound will be computed with p= .9 alpha-1=.95 + else: + distribution = x.values[3] + distribution = distribution.reshape(len(distribution), 1) + + mu = np.array([x.mu]) + + s = sd.significant_digits( + distribution, + mu, + precision=sd.Precision.Relative, + method=sd.Method.CNH, + + probability=0.9, + confidence=0.95 + ) + + return s[0] + + +########################################################################## + + # Main functions + + +# Open and read the tests config file +def read_config(): + try: + with open("vfc_tests_config.json", "r") as file: + data = file.read() + + except FileNotFoundError as e: + e.strerror = "Error [vfc_ci]: This file is required to describe the tests "\ + "to run and generate a Verificarlo run file" + raise e + + return json.loads(data) + + +# Set up metadata +def generate_metadata(is_git_commit): + + # Metadata and filename are initiated as if no commit was associated + metadata = { + "timestamp": calendar.timegm(time.gmtime()), + "is_git_commit": is_git_commit, + "hash": "", + "author": "", + "message": "" + } + + if is_git_commit: + print("Fetching metadata from last commit...") + from git import Repo + + repo = Repo(".") + head_commit = repo.head.commit + + metadata["timestamp"] = head_commit.authored_date + + metadata["hash"] = str(head_commit)[0:7] + metadata["author"] = "%s <%s>" \ + % (str(head_commit.author), head_commit.author.email) + metadata["message"] = head_commit.message.split("\n")[0] + + return metadata + + +# Execute tests and collect results in a Pandas dataframe (+ dataprocessing) +def run_tests(config): + + # Run the build command + print("Info [vfc_ci]: Building tests...") + os.system(config["make_command"]) + + # This is an array of Pandas dataframes for now + data = [] + + # Create tmp folder to export results + os.system("mkdir .vfcruns.tmp") + n_files = 0 + + # This will contain all executables/repetition numbers from which we could + # not get any data + warnings = [] + + # Tests execution loop + for executable in config["executables"]: + print("Info [vfc_ci]: Running executable :", + executable["executable"], "...") + + parameters = "" + if "parameters" in executable: + parameters = executable["parameters"] + + for backend in executable["vfc_backends"]: + + export_backend = "VFC_BACKENDS=\"" + backend["name"] + "\" " + command = "./" + executable["executable"] + " " + parameters + + repetitions = 1 + if "repetitions" in backend: + repetitions = backend["repetitions"] + + # Run test repetitions and save results + for i in range(repetitions): + file = ".vfcruns.tmp/%s.csv" % str(n_files) + export_output = "VFC_PROBES_OUTPUT=\"%s\" " % file + os.system(export_output + export_backend + command) + + # This will only be used if we need to append this exec to the + # warnings list + execution_data = { + "executable": executable["executable"], + "backend": backend["name"], + "repetition": i + 1 + } + + data.append(read_probes_csv( + file, + backend["name"], + warnings, + execution_data + )) + + n_files = n_files + 1 + + # Clean CSV output files (by deleting the tmp folder) + os.system("rm -rf .vfcruns.tmp") + + # Combine all separate executions in one dataframe + data = pd.concat(data, sort=False, ignore_index=True) + data = data.groupby(["test", "vfc_backend", "variable"])\ + .values.apply(list).reset_index() + + # Make sure we have some data to work on + assert(len(data) != 0), "Error [vfc_ci]: No data have been generated " \ + "by your tests executions, aborting run without writing results file" + + return data, warnings + + # Data processing + + +def data_processing(data): + + data["values"] = data["values"].apply(lambda x: np.array(x).astype(float)) + + # Get empirical average, standard deviation and p-value + data["mu"] = data["values"].apply(np.average) + data["sigma"] = data["values"].apply(np.std) + data["pvalue"] = data["values"].apply( + lambda x: scipy.stats.shapiro(x).pvalue) + + # Significant digits + data["s2"] = data.apply(significant_digits, axis=1) + data["s10"] = data["s2"].apply(lambda x: sd.change_base(x, 10)) + + # Lower bound of the confidence interval using the sigdigits module + data["s2_lower_bound"] = data.apply(significant_digits_lower_bound, axis=1) + data["s10_lower_bound"] = data["s2_lower_bound"].apply( + lambda x: sd.change_base(x, 10)) + + # Compute moments of the distribution + # (including a new distribution obtained by filtering outliers) + data["values"] = data["values"].apply(np.sort) + + data["mu"] = data["values"].apply(np.average) + data["min"] = data["values"].apply(np.min) + data["quantile25"] = data["values"].apply(np.quantile, args=(0.25,)) + data["quantile50"] = data["values"].apply(np.quantile, args=(0.50,)) + data["quantile75"] = data["values"].apply(np.quantile, args=(0.75,)) + data["max"] = data["values"].apply(np.max) + + data["nsamples"] = data["values"].apply(len) + + # Display all executions that resulted in a warning + + +def show_warnings(warnings): + if len(warnings) > 0: + print( + "Warning [vfc_ci]: Some of your runs could not generate any data " + "(for instance because your code crashed) and resulted in " + "warnings. Here is the complete list :" + ) + + for i in range(0, len(warnings)): + print("- Warning %s:" % i) + + print(" Executable: %s" % warnings[i]["executable"]) + print(" Backend: %s" % warnings[i]["backend"]) + print(" Repetition: %s" % warnings[i]["repetition"]) + + +########################################################################## + + # Entry point + +def run(is_git_commit, export_raw_values, dry_run): + + # Get config, metadata and data + print("Info [vfc_ci]: Reading tests config file...") + config = read_config() + + print("Info [vfc_ci]: Generating run metadata...") + metadata = generate_metadata(is_git_commit) + + data, warnings = run_tests(config) + show_warnings(warnings) + + # Data processing + print("Info [vfc_ci]: Processing data...") + data_processing(data) + + # Prepare data for export (by creating a proper index and linking run + # timestamp) + data = data.set_index(["test", "variable", "vfc_backend"]).sort_index() + data["timestamp"] = metadata["timestamp"] + + filename = metadata["hash"] if is_git_commit else str( + metadata["timestamp"]) + + # Prepare metadata for export + metadata = pd.DataFrame.from_dict([metadata]) + metadata = metadata.set_index("timestamp") + + # NOTE : Exporting to HDF5 requires to install "tables" on the system + + # Export raw data if needed + if export_raw_values and not dry_run: + data.to_hdf(filename + ".vfcraw.h5", key="data") + metadata.to_hdf(filename + ".vfcraw.h5", key="metadata") + + # Export data + del data["values"] + if not dry_run: + data.to_hdf(filename + ".vfcrun.h5", key="data") + metadata.to_hdf(filename + ".vfcrun.h5", key="metadata") + + # Print termination messages + print( + "Info [vfc_ci]: The results have been successfully written to " + "%s.vfcrun.h5." + % filename + ) + + if export_raw_values: + print( + "Info [vfc_ci]: A file containing the raw values has also been " + "created : %s.vfcraw.h5." + % filename + ) + + if dry_run: + print( + "Info [vfc_ci]: The dry run flag was enabled, so no files were " + "actually created." + ) diff --git a/ci/vfc_ci_report/compare_runs.py b/ci/vfc_ci_report/compare_runs.py new file mode 100644 index 0000000..bc8da4e --- /dev/null +++ b/ci/vfc_ci_report/compare_runs.py @@ -0,0 +1,542 @@ +# Manage the view comparing a variable over different runs + +import time + +import pandas as pd + +from math import pi + +from bokeh.plotting import figure, curdoc +from bokeh.embed import components +from bokeh.models import Select, ColumnDataSource, Panel, Tabs, HoverTool, \ + TextInput, CheckboxGroup, TapTool, CustomJS + +import helper +import plot + + +########################################################################## + + +class CompareRuns: + + # Helper functions related to CompareRuns + + # From an array of timestamps, returns the array of runs names (for the x + # axis ticks), as well as the metadata (in a dict of arrays) associated to + # this array (for the tooltips) + def gen_x_series(self, timestamps): + + # Initialize the objects to return + x_series = [] + x_metadata = dict( + date=[], + is_git_commit=[], + hash=[], + author=[], + message=[] + ) + + # n == 0 means we want all runs, we also make sure not to go out of + # bound if asked for more runs than we have + n = self.current_n_runs + if n == 0 or n > len(timestamps): + n = len(timestamps) + + for i in range(0, n): + # Get metadata associated to this run + row_metadata = helper.get_metadata( + self.metadata, timestamps[-i - 1]) + date = time.ctime(timestamps[-i - 1]) + + # Fill the x series + str = row_metadata["name"] + x_series.insert(0, helper.get_metadata( + self.metadata, timestamps[-i - 1])["name"]) + + # Fill the metadata lists + x_metadata["date"].insert(0, date) + x_metadata["is_git_commit"].insert( + 0, row_metadata["is_git_commit"]) + x_metadata["hash"].insert(0, row_metadata["hash"]) + x_metadata["author"].insert(0, row_metadata["author"]) + x_metadata["message"].insert(0, row_metadata["message"]) + + return x_series, x_metadata + + # Plots update function + + def update_plots(self): + + # Select all data matching current test/var/backend + + runs = self.data.loc[[self.widgets["select_test"].value], + self.widgets["select_var"].value, + self.widgets["select_backend"].value] + + timestamps = runs["timestamp"] + x_series, x_metadata = self.gen_x_series(timestamps.sort_values()) + + # Update source + + main_dict = runs.to_dict("series") + main_dict["x"] = x_series + + # Add metadata (for tooltip) + main_dict.update(x_metadata) + + # Select the last n runs only + n = self.current_n_runs + main_dict = {key: value[-n:] for key, value in main_dict.items()} + + # Generate ColumnDataSources for the 3 dotplots + for stat in ["sigma", "s10", "s2"]: + dict = { + "%s_x" % stat: main_dict["x"], + + "is_git_commit": main_dict["is_git_commit"], + "date": main_dict["date"], + "hash": main_dict["hash"], + "author": main_dict["author"], + "message": main_dict["message"], + + stat: main_dict[stat], + + "nsamples": main_dict["nsamples"], + } + + if stat == "s10" or stat == "s2": + dict["%s_lower_bound" % + stat] = main_dict["%s_lower_bound" % + stat] + + # Filter outliers if the box is checked + if len(self.widgets["outliers_filtering_compare"].active) > 0: + outliers = helper.detect_outliers(dict[stat]) + dict[stat] = helper.remove_outliers(dict[stat], outliers) + dict["%s_x" % stat] = helper.remove_outliers( + dict["%s_x" % stat], outliers) + + # Assign ColumnDataSource + self.sources["%s_source" % stat].data = dict + + # Generate ColumnDataSource for the boxplot + dict = { + "is_git_commit": main_dict["is_git_commit"], + "date": main_dict["date"], + "hash": main_dict["hash"], + "author": main_dict["author"], + "message": main_dict["message"], + + "x": main_dict["x"], + "min": main_dict["min"], + "quantile25": main_dict["quantile25"], + "quantile50": main_dict["quantile50"], + "quantile75": main_dict["quantile75"], + "max": main_dict["max"], + "mu": main_dict["mu"], + "pvalue": main_dict["pvalue"], + + "nsamples": main_dict["nsamples"] + } + + self.sources["boxplot_source"].data = dict + + # Update x axis + + helper.reset_x_range( + self.plots["boxplot"], + self.sources["boxplot_source"].data["x"] + ) + helper.reset_x_range( + self.plots["sigma_plot"], + self.sources["sigma_source"].data["sigma_x"] + ) + helper.reset_x_range( + self.plots["s10_plot"], + self.sources["s10_source"].data["s10_x"] + ) + helper.reset_x_range( + self.plots["s2_plot"], + self.sources["s2_source"].data["s2_x"] + ) + + # Widgets' callback functions + + def update_test(self, attrname, old, new): + + # If the value is updated by the CustomJS, self.widgets["select_var"].value + # won't be updated, so we have to look for that case and assign it + # manually + + # "new" should be a list when updated by CustomJS + if isinstance(new, list): + # If filtering removed all options, we might have an empty list + # (in this case, we just skip the callback and do nothing) + if len(new) > 0: + new = new[0] + else: + return + + if new != self.widgets["select_test"].value: + # The callback will be triggered again with the updated value + self.widgets["select_test"].value = new + return + + # New list of available vars + self.vars = self.data.loc[new]\ + .index.get_level_values("variable").drop_duplicates().tolist() + self.widgets["select_var"].options = self.vars + + # Reset var selection if old one is not available in new vars + if self.widgets["select_var"].value not in self.vars: + self.widgets["select_var"].value = self.vars[0] + # The update_var callback will be triggered by the assignment + + else: + # Trigger the callback manually (since the plots need to be updated + # anyway) + self.update_var("", "", self.widgets["select_var"].value) + + def update_var(self, attrname, old, new): + + # If the value is updated by the CustomJS, self.widgets["select_var"].value + # won't be updated, so we have to look for that case and assign it + # manually + + # new should be a list when updated by CustomJS + if isinstance(new, list): + new = new[0] + + if new != self.widgets["select_var"].value: + # The callback will be triggered again with the updated value + self.widgets["select_var"].value = new + return + + # New list of available backends + self.backends = self.data.loc[self.widgets["select_test"].value, self.widgets["select_var"].value]\ + .index.get_level_values("vfc_backend").drop_duplicates().tolist() + self.widgets["select_backend"].options = self.backends + + # Reset backend selection if old one is not available in new backends + if self.widgets["select_backend"].value not in self.backends: + self.widgets["select_backend"].value = self.backends[0] + # The update_backend callback will be triggered by the assignment + + else: + # Trigger the callback manually (since the plots need to be updated + # anyway) + self.update_backend("", "", self.widgets["select_backend"].value) + + def update_backend(self, attrname, old, new): + + # Simply update plots, since no other data is affected + self.update_plots() + + def update_n_runs(self, attrname, old, new): + # Simply update runs selection (value and string display) + self.select_n_runs.value = new + self.current_n_runs = self.n_runs_dict[self.select_n_runs.value] + + self.update_plots() + + def update_outliers_filtering(self, attrname, old, new): + self.update_plots() + + # Bokeh setup functions + + def setup_plots(self): + + tools = "pan, wheel_zoom, xwheel_zoom, ywheel_zoom, reset, save" + + # Custom JS callback that will be used when tapping on a run + # Only switches the view, a server callback is required to update plots + # (defined inside template to avoid bloating server w/ too much JS code) + js_tap_callback = "goToInspectRuns();" + + # Box plot + self.plots["boxplot"] = figure( + name="boxplot", title="Variable distribution over runs", + plot_width=900, plot_height=400, x_range=[""], + tools=tools, sizing_mode="scale_width" + ) + + box_tooltips = [ + ("Git commit", "@is_git_commit"), + ("Date", "@date"), + ("Hash", "@hash"), + ("Author", "@author"), + ("Message", "@message"), + ("Min", "@min{%0.18e}"), + ("Max", "@max{%0.18e}"), + ("1st quartile", "@quantile25{%0.18e}"), + ("Median", "@quantile50{%0.18e}"), + ("3rd quartile", "@quantile75{%0.18e}"), + ("μ", "@mu{%0.18e}"), + ("p-value", "@pvalue"), + ("Number of samples", "@nsamples") + ] + box_tooltips_formatters = { + "@min": "printf", + "@max": "printf", + "@quantile25": "printf", + "@quantile50": "printf", + "@quantile75": "printf", + "@mu": "printf" + } + + plot.fill_boxplot( + self.plots["boxplot"], self.sources["boxplot_source"], + tooltips=box_tooltips, + tooltips_formatters=box_tooltips_formatters, + js_tap_callback=js_tap_callback, + server_tap_callback=self.inspect_run_callback_boxplot, + ) + self.doc.add_root(self.plots["boxplot"]) + + # Sigma plot (bar plot) + self.plots["sigma_plot"] = figure( + name="sigma_plot", title="Standard deviation σ over runs", + plot_width=900, plot_height=400, x_range=[""], + tools=tools, sizing_mode="scale_width" + ) + + sigma_tooltips = [ + ("Git commit", "@is_git_commit"), + ("Date", "@date"), + ("Hash", "@hash"), + ("Author", "@author"), + ("Message", "@message"), + ("σ", "@sigma"), + ("Number of samples", "@nsamples") + ] + + plot.fill_dotplot( + self.plots["sigma_plot"], self.sources["sigma_source"], "sigma", + tooltips=sigma_tooltips, + js_tap_callback=js_tap_callback, + server_tap_callback=self.inspect_run_callback_sigma, + lines=True + ) + self.doc.add_root(self.plots["sigma_plot"]) + + # s plot (bar plot with 2 tabs) + self.plots["s10_plot"] = figure( + name="s10_plot", title="Significant digits s over runs", + plot_width=900, plot_height=400, x_range=[""], + tools=tools, sizing_mode="scale_width" + ) + + s10_tooltips = [ + ("Git commit", "@is_git_commit"), + ("Date", "@date"), + ("Hash", "@hash"), + ("Author", "@author"), + ("Message", "@message"), + ("s", "@s10"), + ("s lower bound", "@s10_lower_bound"), + ("Number of samples", "@nsamples") + ] + + plot.fill_dotplot( + self.plots["s10_plot"], self.sources["s10_source"], "s10", + tooltips=s10_tooltips, + js_tap_callback=js_tap_callback, + server_tap_callback=self.inspect_run_callback_s10, + lines=True, + lower_bound=True + ) + s10_tab = Panel(child=self.plots["s10_plot"], title="Base 10") + + self.plots["s2_plot"] = figure( + name="s2_plot", title="Significant digits s over runs", + plot_width=900, plot_height=400, x_range=[""], + tools=tools, sizing_mode="scale_width" + ) + + s2_tooltips = [ + ("Git commit", "@is_git_commit"), + ("Date", "@date"), + ("Hash", "@hash"), + ("Author", "@author"), + ("Message", "@message"), + ("s", "@s2"), + ("s lower bound", "@s2_lower_bound"), + ("Number of samples", "@nsamples") + ] + + plot.fill_dotplot( + self.plots["s2_plot"], self.sources["s2_source"], "s2", + tooltips=s2_tooltips, + js_tap_callback=js_tap_callback, + server_tap_callback=self.inspect_run_callback_s2, + lines=True, + lower_bound=True + ) + s2_tab = Panel(child=self.plots["s2_plot"], title="Base 2") + + s_tabs = Tabs( + name="s_tabs", + tabs=[s10_tab, s2_tab], + tabs_location="below" + ) + + self.doc.add_root(s_tabs) + + def setup_widgets(self): + + # Initial selections + + # Test/var/backend combination (we select all first elements at init) + self.tests = self.data\ + .index.get_level_values("test").drop_duplicates().tolist() + + self.vars = self.data.loc[self.tests[0]]\ + .index.get_level_values("variable").drop_duplicates().tolist() + + self.backends = self.data.loc[self.tests[0], self.vars[0]]\ + .index.get_level_values("vfc_backend").drop_duplicates().tolist() + + # Custom JS callback that will be used client side to filter selections + filter_callback_js = """ + selector.options = options.filter(e => e.includes(cb_obj.value)); + """ + + # Test selector widget + + # Number of runs to display + # The dict structure allows us to get int value from the display string + # in O(1) + self.n_runs_dict = { + "Last 3 runs": 3, + "Last 5 runs": 5, + "Last 10 runs": 10, + "All runs": 0 + } + + # Contains all options strings + n_runs_display = list(self.n_runs_dict.keys()) + + # Will be used when updating plots (contains actual number to diplay) + self.current_n_runs = self.n_runs_dict[n_runs_display[1]] + + # Selector widget + self.widgets["select_test"] = Select( + name="select_test", title="Test :", + value=self.tests[0], options=self.tests + ) + self.doc.add_root(self.widgets["select_test"]) + self.widgets["select_test"].on_change("value", self.update_test) + self.widgets["select_test"].on_change("options", self.update_test) + + # Filter widget + self.widgets["test_filter"] = TextInput( + name="test_filter", title="Tests filter:" + ) + self.widgets["test_filter"].js_on_change( + "value", + CustomJS( + args=dict( + options=self.tests, + selector=self.widgets["select_test"]), + code=filter_callback_js)) + self.doc.add_root(self.widgets["test_filter"]) + + # Number of runs to display + + self.widgets["select_n_runs"] = Select( + name="select_n_runs", title="Display :", + value=n_runs_display[1], options=n_runs_display + ) + self.doc.add_root(self.widgets["select_n_runs"]) + self.widgets["select_n_runs"].on_change("value", self.update_n_runs) + + # Variable selector widget + + self.widgets["select_var"] = Select( + name="select_var", title="Variable :", + value=self.vars[0], options=self.vars + ) + self.doc.add_root(self.widgets["select_var"]) + self.widgets["select_var"].on_change("value", self.update_var) + self.widgets["select_var"].on_change("options", self.update_var) + + # Backend selector widget + + self.widgets["select_backend"] = Select( + name="select_backend", title="Verificarlo backend :", + value=self.backends[0], options=self.backends + ) + self.doc.add_root(self.widgets["select_backend"]) + self.widgets["select_backend"].on_change("value", self.update_backend) + + # Outliers filtering checkbox + + self.widgets["outliers_filtering_compare"] = CheckboxGroup( + name="outliers_filtering_compare", + labels=["Filter outliers"], active=[] + ) + self.doc.add_root(self.widgets["outliers_filtering_compare"]) + self.widgets["outliers_filtering_compare"]\ + .on_change("active", self.update_outliers_filtering) + + # Communication methods + # (to send/receive messages to/from master) + + # Callback to change view of Inspect runs when data is selected + + def inspect_run_callback(self, new, source_name, x_name): + + # In case we just unselected everything, then do nothing + if new == []: + return + + index = new[-1] + run_name = self.sources[source_name].data[x_name][index] + + self.master.go_to_inspect(run_name) + + # Wrappers for each plot (since new is the index of the clicked element, + # it is dependent of the plot because we could have filtered some outliers) + # There doesn't seem to be an easy way to add custom parameters to a + # Bokeh callback, so using wrappers seems to be the best solution for now + + def inspect_run_callback_boxplot(self, attr, old, new): + self.inspect_run_callback(new, "boxplot_source", "x") + + def inspect_run_callback_sigma(self, attr, old, new): + self.inspect_run_callback(new, "sigma_source", "sigma_x") + + def inspect_run_callback_s2(self, attr, old, new): + self.inspect_run_callback(new, "s2_source", "s2_x") + + def inspect_run_callback_s10(self, attr, old, new): + self.inspect_run_callback(new, "s10_source", "s10_x") + + # Constructor + + def __init__(self, master, doc, data, metadata): + + self.master = master + + self.doc = doc + self.data = data + self.metadata = metadata + + self.sources = { + "boxplot_source": ColumnDataSource(data={}), + "sigma_source": ColumnDataSource(data={}), + "s10_source": ColumnDataSource(data={}), + "s2_source": ColumnDataSource(data={}) + } + + self.plots = {} + self.widgets = {} + + # Setup Bokeh objects + self.setup_plots() + self.setup_widgets() + + # At this point, everything should have been initialized, so we can + # show the plots for the first time + self.update_plots() diff --git a/ci/vfc_ci_report/helper.py b/ci/vfc_ci_report/helper.py new file mode 100644 index 0000000..f258959 --- /dev/null +++ b/ci/vfc_ci_report/helper.py @@ -0,0 +1,170 @@ +# General helper functions for both compare_runs and compare_variables + +import calendar +import time +from itertools import compress + +import numpy as np + +# Magic numbers +max_ticks = 15 +max_zscore = 3 + +########################################################################## + + +# From a timestamp, return the associated metadata as a Pandas serie +def get_metadata(metadata, timestamp): + return metadata.loc[timestamp] + + +# Convert a metadata Pandas series to a JS readable dict +def metadata_to_dict(metadata): + dict = metadata.to_dict() + + # JS doesn't accept True for booleans, and Python doesn't accept true + # (because of the caps) => using an integer is a portable solution + dict["is_git_commit"] = 1 if dict["is_git_commit"] else 0 + + dict["date"] = time.ctime(metadata.name) + + return dict + + +# Return a string that indicates the elapsed time since the run, used as the +# x-axis tick in "Compare runs" or when selecting run in "Inspect run" +def get_run_name(timestamp, hash): + + gmt = time.gmtime() + now = calendar.timegm(gmt) + diff = now - timestamp + + # Special case : < 1 minute (return string directly) + if diff < 60: + str = "Less than a minute ago" + + if hash != "": + str = str + " (%s)" % hash + + if str == get_run_name.previous: + get_run_name.counter = get_run_name.counter + 1 + str = "%s (%s)" % (str, get_run_name.counter) + else: + get_run_name.counter = 0 + get_run_name.previous = str + + return str + + # < 1 hour + if diff < 3600: + n = int(diff / 60) + str = "%s minute%s ago" + # < 1 day + elif diff < 86400: + n = int(diff / 3600) + str = "%s hour%s ago" + # < 1 week + elif diff < 604800: + n = int(diff / 86400) + str = "%s day%s ago" + # < 1 month + elif diff < 2592000: + n = int(diff / 604800) + str = "%s week%s ago" + # > 1 month + else: + n = diff / 2592000 + str = "%s month%s ago" + + plural = "" + if n != 1: + plural = "s" + + str = str % (n, plural) + + # We might want to add the git hash + if hash != "": + str = str + " (%s)" % hash + + # Finally, check for duplicate with previously generated string + if str == get_run_name.previous: + # Increment the duplicate counter and add it to str + get_run_name.counter = get_run_name.counter + 1 + str = "%s (%s)" % (str, get_run_name.counter) + + else: + # No duplicate, reset both previously generated str and duplicate + # counter + get_run_name.counter = 0 + get_run_name.previous = str + + return str + + +# These external variables will store data about the last generated string to +# avoid duplicates (assuming the runs are sorted by time) +get_run_name.counter = 0 +get_run_name.previous = "" + + +def reset_run_strings(): + get_run_name.counter = 0 + get_run_name.previous = "" + + +# Update all the x-ranges from a dict of plots +def reset_x_range(plot, x_range): + plot.x_range.factors = x_range + + if len(x_range) < max_ticks: + plot.xaxis.major_tick_line_color = "#000000" + plot.xaxis.minor_tick_line_color = "#000000" + + plot.xaxis.major_label_text_font_size = "8pt" + + else: + plot.xaxis.major_tick_line_color = None + plot.xaxis.minor_tick_line_color = None + + plot.xaxis.major_label_text_font_size = "0pt" + + +# Return an array of booleans that indicate which elements are outliers +# (True means element is not an outlier and must be kept) +def detect_outliers(array, max_zscore=max_zscore): + if len(array) <= 2: + return [True] * len(array) + + median = np.median(array) + std = np.std(array) + if std == 0: + return array + distance = abs(array - median) + # Array of booleans with elements to be filtered + outliers_array = distance < max_zscore * std + + return outliers_array + + +def remove_outliers(array, outliers): + return list(compress(array, outliers)) + + +def remove_boxplot_outliers(dict, outliers, prefix): + outliers = detect_outliers(dict["%s_max" % prefix]) + + dict["%s_x" % prefix] = remove_outliers(dict["%s_x" % prefix], outliers) + + dict["%s_min" % prefix] = remove_outliers( + dict["%s_min" % prefix], outliers) + dict["%s_quantile25" % prefix] = remove_outliers( + dict["%s_quantile25" % prefix], outliers) + dict["%s_quantile50" % prefix] = remove_outliers( + dict["%s_quantile50" % prefix], outliers) + dict["%s_quantile75" % prefix] = remove_outliers( + dict["%s_quantile75" % prefix], outliers) + dict["%s_max" % prefix] = remove_outliers( + dict["%s_max" % prefix], outliers) + dict["%s_mu" % prefix] = remove_outliers(dict["%s_mu" % prefix], outliers) + + dict["nsamples"] = remove_outliers(dict["nsamples"], outliers) diff --git a/ci/vfc_ci_report/inspect_runs.py b/ci/vfc_ci_report/inspect_runs.py new file mode 100644 index 0000000..f53ab17 --- /dev/null +++ b/ci/vfc_ci_report/inspect_runs.py @@ -0,0 +1,570 @@ +# Manage the view comparing the variables of a run + +from math import pi +from functools import partial + +import pandas as pd +import numpy as np + +from bokeh.plotting import figure, curdoc +from bokeh.embed import components +from bokeh.models import Select, ColumnDataSource, Panel, Tabs, HoverTool,\ + RadioButtonGroup, CheckboxGroup, CustomJS + +import helper +import plot + + +########################################################################## + + +class InspectRuns: + + # Helper functions related to InspectRun + + # Returns a dictionary mapping user-readable strings to all run timestamps + def gen_runs_selection(self): + + runs_dict = {} + + # Iterate over timestamp rows (runs) and fill dict + for row in self.metadata.iloc: + # The syntax used by pandas makes this part a bit tricky : + # row.name is the index of metadata (so it refers to the + # timestamp), whereas row["name"] is the column called "name" + # (which is the display string used for the run) + + # runs_dict[run's name] = run's timestamp + runs_dict[row["name"]] = row.name + + return runs_dict + + def gen_boxplot_tooltips(self, prefix): + return [ + ("Name", "@%s_x" % prefix), + ("Min", "@" + prefix + "_min{%0.18e}"), + ("Max", "@" + prefix + "_max{%0.18e}"), + ("1st quartile", "@" + prefix + "_quantile25{%0.18e}"), + ("Median", "@" + prefix + "_quantile50{%0.18e}"), + ("3rd quartile", "@" + prefix + "_quantile75{%0.18e}"), + ("μ", "@" + prefix + "_mu{%0.18e}"), + ("Number of samples (tests)", "@nsamples") + ] + + def gen_boxplot_tooltips_formatters(self, prefix): + return { + "@%s_min" % prefix: "printf", + "@%s_max" % prefix: "printf", + "@%s_quantile25" % prefix: "printf", + "@%s_quantile50" % prefix: "printf", + "@%s_quantile75" % prefix: "printf", + "@%s_mu" % prefix: "printf" + } + + # Data processing helper + # (computes new distributions for sigma, s2, s10) + + def data_processing(self, dataframe): + + # Compute aggragated mu + dataframe["mu"] = np.vectorize( + np.average)( + dataframe["mu"], + weights=dataframe["nsamples"]) + + # nsamples is the number of aggregated elements (as well as the number + # of samples for our new sigma and s distributions) + dataframe["nsamples"] = dataframe["nsamples"].apply(lambda x: len(x)) + + dataframe["mu_x"] = dataframe.index + # Make sure that strings don't excede a certain length + dataframe["mu_x"] = dataframe["mu_x"].apply( + lambda x: x[:17] + "[...]" + x[-17:] if len(x) > 39 else x + ) + + # Get quantiles and mu for sigma, s10, s2 + for prefix in ["sigma", "s10", "s2"]: + + dataframe["%s_x" % prefix] = dataframe["mu_x"] + + dataframe[prefix] = dataframe[prefix].apply(np.sort) + + dataframe["%s_min" % prefix] = dataframe[prefix].apply(np.min) + dataframe["%s_quantile25" % prefix] = dataframe[prefix].apply( + np.quantile, args=(0.25,)) + dataframe["%s_quantile50" % prefix] = dataframe[prefix].apply( + np.quantile, args=(0.50,)) + dataframe["%s_quantile75" % prefix] = dataframe[prefix].apply( + np.quantile, args=(0.75,)) + dataframe["%s_max" % prefix] = dataframe[prefix].apply(np.max) + dataframe["%s_mu" % prefix] = dataframe[prefix].apply(np.average) + del dataframe[prefix] + + return dataframe + + # Plots update function + + def update_plots(self): + + groupby_display = self.widgets["groupby_radio"].labels[ + self.widgets["groupby_radio"].active + ] + groupby = self.factors_dict[groupby_display] + + filterby_display = self.widgets["filterby_radio"].labels[ + self.widgets["filterby_radio"].active + ] + filterby = self.factors_dict[filterby_display] + + # Groupby and aggregate lines belonging to the same group in lists + + groups = self.run_data[ + self.run_data.index.isin( + [self.widgets["select_filter"].value], + level=filterby + ) + ].groupby(groupby) + + groups = groups.agg({ + "sigma": lambda x: x.tolist(), + "s10": lambda x: x.tolist(), + "s2": lambda x: x.tolist(), + + "mu": lambda x: x.tolist(), + + # Used for mu weighted average first, then will be replaced + "nsamples": lambda x: x.tolist() + }) + + # Compute the new distributions, ... + groups = self.data_processing(groups).to_dict("list") + + # Update source + + # Assign each ColumnDataSource, starting with the boxplots + for prefix in ["sigma", "s10", "s2"]: + + dict = { + "%s_x" % prefix: groups["%s_x" % prefix], + "%s_min" % prefix: groups["%s_min" % prefix], + "%s_quantile25" % prefix: groups["%s_quantile25" % prefix], + "%s_quantile50" % prefix: groups["%s_quantile50" % prefix], + "%s_quantile75" % prefix: groups["%s_quantile75" % prefix], + "%s_max" % prefix: groups["%s_max" % prefix], + "%s_mu" % prefix: groups["%s_mu" % prefix], + + "nsamples": groups["nsamples"] + } + + # Filter outliers if the box is checked + if len(self.widgets["outliers_filtering_inspect"].active) > 0: + + # Boxplots will be filtered by max then min + top_outliers = helper.detect_outliers(dict["%s_max" % prefix]) + helper.remove_boxplot_outliers(dict, top_outliers, prefix) + + bottom_outliers = helper.detect_outliers( + dict["%s_min" % prefix]) + helper.remove_boxplot_outliers(dict, bottom_outliers, prefix) + + self.sources["%s_source" % prefix].data = dict + + # Finish with the mu plot + dict = { + "mu_x": groups["mu_x"], + "mu": groups["mu"], + + "nsamples": groups["nsamples"] + } + + self.sources["mu_source"].data = dict + + # Filter outliers if the box is checked + if len(self.widgets["outliers_filtering_inspect"].active) > 0: + mu_outliers = helper.detect_outliers(groups["mu"]) + groups["mu"] = helper.remove_outliers(groups["mu"], mu_outliers) + groups["mu_x"] = helper.remove_outliers( + groups["mu_x"], mu_outliers) + + # Update plots axis/titles + + # Get display string of the last (unselected) factor + factors_dict = self.factors_dict.copy() + del factors_dict[groupby_display] + del factors_dict[filterby_display] + for_all = list(factors_dict.keys())[0] + + # Update all display strings for plot title (remove caps, plural) + groupby_display = groupby_display.lower() + filterby_display = filterby_display.lower()[:-1] + for_all = for_all.lower() + + self.plots["mu_inspect"].title.text = \ + "Empirical average μ of %s (groupped by %s, for all %s)" \ + % (filterby_display, groupby_display, for_all) + + self.plots["sigma_inspect"].title.text = \ + "Standard deviation σ of %s (groupped by %s, for all %s)" \ + % (filterby_display, groupby_display, for_all) + + self.plots["s10_inspect"].title.text = \ + "Significant digits s of %s (groupped by %s, for all %s)" \ + % (filterby_display, groupby_display, for_all) + + self.plots["s2_inspect"].title.text = \ + "Significant digits s of %s (groupped by %s, for all %s)" \ + % (filterby_display, groupby_display, for_all) + + helper.reset_x_range(self.plots["mu_inspect"], groups["mu_x"]) + helper.reset_x_range(self.plots["sigma_inspect"], groups["sigma_x"]) + helper.reset_x_range(self.plots["s10_inspect"], groups["s10_x"]) + helper.reset_x_range(self.plots["s2_inspect"], groups["s2_x"]) + + # Widets' callback functions + + # Run selector callback + + def update_run(self, attrname, old, new): + + filterby = self.widgets["filterby_radio"].labels[ + self.widgets["filterby_radio"].active + ] + filterby = self.factors_dict[filterby] + + # Update run selection (by using dict mapping) + self.current_run = self.runs_dict[new] + + # Update run data + self.run_data = self.data[self.data["timestamp"] == self.current_run] + + # Save old selected option + old_value = self.widgets["select_filter"].value + + # Update filter options + options = self.run_data.index\ + .get_level_values(filterby).drop_duplicates().tolist() + self.widgets["select_filter"].options = options + + if old_value not in self.widgets["select_filter"].options: + self.widgets["select_filter"].value = options[0] + # The update_var callback will be triggered by the assignment + + else: + # Trigger the callback manually (since the plots need to be updated + # anyway) + self.update_filter("", "", old_value) + + # "Group by" radio + + def update_groupby(self, attrname, old, new): + + # Update "Filter by" radio list + filterby_list = list(self.factors_dict.keys()) + del filterby_list[self.widgets["groupby_radio"].active] + self.widgets["filterby_radio"].labels = filterby_list + + filterby = self.widgets["filterby_radio"].labels[ + self.widgets["filterby_radio"].active + ] + filterby = self.factors_dict[filterby] + + # Save old selected option + old_value = self.widgets["select_filter"].value + + # Update filter options + options = self.run_data.index\ + .get_level_values(filterby).drop_duplicates().tolist() + self.widgets["select_filter"].options = options + + if old_value not in self.widgets["select_filter"].options: + self.widgets["select_filter"].value = options[0] + # The update_var callback will be triggered by the assignment + + else: + # Trigger the callback manually (since the plots need to be updated + # anyway) + self.update_filter("", "", old_value) + + # "Filter by" radio + + def update_filterby(self, attrname, old, new): + + filterby = self.widgets["filterby_radio"].labels[ + self.widgets["filterby_radio"].active + ] + filterby = self.factors_dict[filterby] + + # Save old selected option + old_value = self.widgets["select_filter"].value + + # Update filter selector options + options = self.run_data.index\ + .get_level_values(filterby).drop_duplicates().tolist() + self.widgets["select_filter"].options = options + + if old_value not in self.widgets["select_filter"].options: + self.widgets["select_filter"].value = options[0] + # The update_var callback will be triggered by the assignment + + else: + # Trigger the callback manually (since the plots need to be updated + # anyway) + self.update_filter("", "", old_value) + + # Filter selector callback + + def update_filter(self, attrname, old, new): + self.update_plots() + + # Filter outliers checkbox callback + + def update_outliers_filtering(self, attrname, old, new): + # The status (checked/unchecked) of the checkbox is also verified inside + # self.update_plots(), so calling this function is enough + self.update_plots() + + # Bokeh setup functions + # (for both variable and backend selection at once) + + def setup_plots(self): + + tools = "pan, wheel_zoom, xwheel_zoom, ywheel_zoom, reset, save" + + # Tooltips and formatters + + dotplot_tooltips = [ + ("Name", "@mu_x"), + ("μ", "@mu{%0.18e}"), + ("Number of samples (tests)", "@nsamples") + ] + dotplot_formatters = { + "@mu": "printf" + } + + sigma_boxplot_tooltips = self.gen_boxplot_tooltips("sigma") + sigma_boxplot_tooltips_formatters = self.gen_boxplot_tooltips_formatters( + "sigma") + + s10_boxplot_tooltips = self.gen_boxplot_tooltips("s10") + s10_boxplot_tooltips_formatters = self.gen_boxplot_tooltips_formatters( + "s10") + + s2_boxplot_tooltips = self.gen_boxplot_tooltips("s2") + s2_boxplot_tooltips_formatters = self.gen_boxplot_tooltips_formatters( + "s2") + + # Plots + + # Mu plot + self.plots["mu_inspect"] = figure( + name="mu_inspect", + title="", + plot_width=900, plot_height=400, x_range=[""], + tools=tools, sizing_mode="scale_width" + ) + plot.fill_dotplot( + self.plots["mu_inspect"], self.sources["mu_source"], "mu", + tooltips=dotplot_tooltips, + tooltips_formatters=dotplot_formatters + ) + self.doc.add_root(self.plots["mu_inspect"]) + + # Sigma plot + self.plots["sigma_inspect"] = figure( + name="sigma_inspect", + title="", + plot_width=900, plot_height=400, x_range=[""], + tools=tools, sizing_mode="scale_width" + ) + plot.fill_boxplot( + self.plots["sigma_inspect"], + self.sources["sigma_source"], + prefix="sigma", + tooltips=sigma_boxplot_tooltips, + tooltips_formatters=sigma_boxplot_tooltips_formatters) + self.doc.add_root(self.plots["sigma_inspect"]) + + # s plots + self.plots["s10_inspect"] = figure( + name="s10_inspect", + title="", + plot_width=900, plot_height=400, x_range=[""], + tools=tools, sizing_mode='scale_width' + ) + plot.fill_boxplot( + self.plots["s10_inspect"], + self.sources["s10_source"], + prefix="s10", + tooltips=s10_boxplot_tooltips, + tooltips_formatters=s10_boxplot_tooltips_formatters) + s10_tab_inspect = Panel( + child=self.plots["s10_inspect"], + title="Base 10") + + self.plots["s2_inspect"] = figure( + name="s2_inspect", + title="", + plot_width=900, plot_height=400, x_range=[""], + tools=tools, sizing_mode='scale_width' + ) + plot.fill_boxplot( + self.plots["s2_inspect"], self.sources["s2_source"], prefix="s2", + tooltips=s2_boxplot_tooltips, + tooltips_formatters=s2_boxplot_tooltips_formatters + ) + s2_tab_inspect = Panel(child=self.plots["s2_inspect"], title="Base 2") + + s_tabs_inspect = Tabs( + name="s_tabs_inspect", + tabs=[s10_tab_inspect, s2_tab_inspect], tabs_location="below" + ) + self.doc.add_root(s_tabs_inspect) + + def setup_widgets(self): + + # Generation of selectable items + + # Dict contains all inspectable runs (maps display strings to timestamps) + # The dict structure allows to get the timestamp from the display string + # in O(1) + self.runs_dict = self.gen_runs_selection() + + # Dict maps display strings to column names for the different factors + # (var, backend, test) + self.factors_dict = { + "Variables": "variable", + "Backends": "vfc_backend", + "Tests": "test" + } + + # Run selection + + # Contains all options strings + runs_display = list(self.runs_dict.keys()) + # Will be used when updating plots (contains actual number) + self.current_run = self.runs_dict[runs_display[-1]] + # Contains the selected option string, used to update current_n_runs + current_run_display = runs_display[-1] + # This contains only entries matching the run + self.run_data = self.data[self.data["timestamp"] == self.current_run] + + change_run_callback_js = "updateRunMetadata(cb_obj.value);" + + self.widgets["select_run"] = Select( + name="select_run", title="Run :", + value=current_run_display, options=runs_display + ) + self.doc.add_root(self.widgets["select_run"]) + self.widgets["select_run"].on_change("value", self.update_run) + self.widgets["select_run"].js_on_change("value", CustomJS( + code=change_run_callback_js, + args=(dict( + metadata=helper.metadata_to_dict( + helper.get_metadata(self.metadata, self.current_run) + ) + )) + )) + + # Factors selection + + # "Group by" radio + self.widgets["groupby_radio"] = RadioButtonGroup( + name="groupby_radio", + labels=list(self.factors_dict.keys()), active=0 + ) + self.doc.add_root(self.widgets["groupby_radio"]) + # The functions are defined inside the template to avoid writing too + # much JS server side + self.widgets["groupby_radio"].on_change( + "active", + self.update_groupby + ) + + # "Filter by" radio + # Get all possible factors, and remove the one selected in "Group by" + filterby_list = list(self.factors_dict.keys()) + del filterby_list[self.widgets["groupby_radio"].active] + + self.widgets["filterby_radio"] = RadioButtonGroup( + name="filterby_radio", + labels=filterby_list, active=0 + ) + self.doc.add_root(self.widgets["filterby_radio"]) + # The functions are defined inside the template to avoid writing too + # much JS server side + self.widgets["filterby_radio"].on_change( + "active", + self.update_filterby + ) + + # Filter selector + + filterby = self.widgets["filterby_radio"].labels[ + self.widgets["filterby_radio"].active + ] + filterby = self.factors_dict[filterby] + + options = self.run_data.index\ + .get_level_values(filterby).drop_duplicates().tolist() + + self.widgets["select_filter"] = Select( + # We need a different name to avoid collision in the template with + # the runs comparison's widget + name="select_filter", title="Select a filter :", + value=options[0], options=options + ) + self.doc.add_root(self.widgets["select_filter"]) + self.widgets["select_filter"]\ + .on_change("value", self.update_filter) + + # Toggle for outliers filtering + + self.widgets["outliers_filtering_inspect"] = CheckboxGroup( + name="outliers_filtering_inspect", + labels=["Filter outliers"], active=[] + ) + self.doc.add_root(self.widgets["outliers_filtering_inspect"]) + self.widgets["outliers_filtering_inspect"]\ + .on_change("active", self.update_outliers_filtering) + + # Communication methods + # (to send/receive messages to/from master) + + # When received, switch to the run_name in parameter + + def switch_view(self, run_name): + self.widgets["select_run"].value = run_name + + # Constructor + + def __init__(self, master, doc, data, metadata): + + self.master = master + + self.doc = doc + self.data = data + self.metadata = metadata + + self.sources = { + "mu_source": ColumnDataSource(data={}), + "sigma_source": ColumnDataSource(data={}), + "s10_source": ColumnDataSource(data={}), + "s2_source": ColumnDataSource(data={}) + } + + self.plots = {} + self.widgets = {} + + # Setup Bokeh objects + self.setup_plots() + self.setup_widgets() + + # Pass the initial metadata to the template (will be updated in CustomJS + # callbacks). This is required because metadata is not displayed in a + # Bokeh widget, so we can't update this with a server callback. + initial_run = helper.get_metadata(self.metadata, self.current_run) + self.doc.template_variables["initial_timestamp"] = self.current_run + + # At this point, everything should have been initialized, so we can + # show the plots for the first time + self.update_plots() diff --git a/ci/vfc_ci_report/main.py b/ci/vfc_ci_report/main.py new file mode 100644 index 0000000..4af73d2 --- /dev/null +++ b/ci/vfc_ci_report/main.py @@ -0,0 +1,208 @@ +# Look for and read all the run files in the current directory (ending with +# .vfcrunh5), and lanch a Bokeh server for the visualization of this data. + +import os +import sys +import time + +import pandas as pd + +from bokeh.plotting import curdoc + +# Local imports from vfc_ci_server +import compare_runs +import inspect_runs +import helper + +########################################################################## + +# Read vfcrun files, and aggregate them in one dataset + +run_files = [f for f in os.listdir(".") if f.endswith(".vfcrun.h5")] + +if len(run_files) == 0: + print( + "Warning [vfc_ci]: Could not find any vfcrun files in the directory. " + "This will result in server errors and prevent you from viewing the report.") + +# These are arrays of Pandas dataframes for now +metadata = [] +data = [] + +for f in run_files: + metadata.append(pd.read_hdf(f, "metadata")) + data.append(pd.read_hdf(f, "data")) + +metadata = pd.concat(metadata).sort_index() +data = pd.concat(data).sort_index() + + +# Generate the display strings for runs (runs ticks) +# By doing this in master, we ensure the homogeneity of display strings +# across all plots +metadata["name"] = metadata.index.to_series().map( + lambda x: helper.get_run_name( + x, + helper.get_metadata(metadata, x)["hash"] + ) +) +helper.reset_run_strings() + +metadata["date"] = metadata.index.to_series().map( + lambda x: time.ctime(x) +) + + +########################################################################## + + +curdoc().title = "Verificarlo Report" + +# Read server arguments +# (this is quite easy because Bokeh server is called through a wrapper, so +# we know exactly what the arguments might be) + +git_repo_linked = False +commit_link = "" + +has_logo = False +logo_url = "" + +for i in range(1, len(sys.argv)): + + # Look for the Git repository remote address + # (if a Git repo is specified, the webpage will contain hyperlinks to the + # repository and the different commits) + if sys.argv[i] == "git": + from urllib.parse import urlparse + + method = sys.argv[i + 1] + address = sys.argv[i + 2] + url = "" + + # Here, address is either the remote URL or the path to the local Git + # repo (depending on the method) + + if method == "url": + # We should directly have a Git URL + url = address + + elif method == "directory": + # Get the remote URL from the local repo + from git import Repo + repo = Repo(address) + url = repo.remotes.origin.url + + else: + raise ValueError( + "Error [vfc_ci]: The specified method to get the Git " + "repository is invalid. Are you calling Bokeh directly " + "instead of using the Verificarlo wrapper ?" + ) + + # At this point, "url" should be set correctly, we can get the repo's + # URL and name, after making sure we're on a Git URL + + parsed_url = urlparse(url) + + path = parsed_url.path.split("/") + if len(path) < 3: + raise ValueError( + "Error [vfc_ci]: The found URL doesn't seem to be pointing " + "to a Git repository (path is too short)" + ) + + repo_name = path[2] + + curdoc().template_variables["repo_url"] = url + curdoc().template_variables["repo_name"] = repo_name + + # We should have a "github.com" or a "*gitlab*" URL + + if parsed_url.netloc == "github.com": + commit_link = "https://%s%s/commit/" \ + % (parsed_url.netloc, parsed_url.path) + + curdoc().template_variables["commit_link"] = commit_link + curdoc().template_variables["git_host"] = "GitHub" + + # Used in Bokeh tooltips + commit_link = commit_link + "@hash" + + # We assume we have a GitLab URL + else: + commit_link = "https://%s%s/-/commit/" \ + % (parsed_url.netloc, parsed_url.path) + + curdoc().template_variables["commit_link"] = commit_link + curdoc().template_variables["git_host"] = "GitLab" + + # Used in Bokeh tooltips + commit_link = commit_link + "@hash" + + git_repo_linked = True + + # Look for a logo URL + # If a logo URL is specified, it will be included in the report's header + if sys.argv[i] == "logo": + curdoc().template_variables["logo_url"] = sys.argv[i + 1] + has_logo = True + + +# After the loop, we know if a repo has been linked, if we have a logo, ... +curdoc().template_variables["git_repo_linked"] = git_repo_linked +curdoc().template_variables["has_logo"] = has_logo + + +########################################################################## + +# Setup report views + +# Define a ViewsMaster class to allow two-ways communication between views. +# This approach by classes allows us to have separate scopes for each view and +# will be useful if we want to add new views at some point in the future +# (instead of having n views with n-1 references each). + +class ViewsMaster: + + # Communication functions + + def go_to_inspect(self, run_name): + self.inspect.switch_view(run_name) + + # Constructor + + def __init__(self, data, metadata, git_repo_linked, commit_link): + + self.data = data + self.metadata = metadata + self.git_repo_linked = git_repo_linked + self.commit_link = commit_link + + # Pass metadata to the template as a JSON string + curdoc().template_variables["metadata"] = self.metadata.to_json( + orient="index") + + # Runs comparison + self.compare = compare_runs.CompareRuns( + master=self, + doc=curdoc(), + data=data, + metadata=metadata, + ) + + # Runs inspection + self.inspect = inspect_runs.InspectRuns( + master=self, + doc=curdoc(), + data=data, + metadata=metadata, + ) + + +views_master = ViewsMaster( + data=data, + metadata=metadata, + git_repo_linked=git_repo_linked, + commit_link=commit_link +) diff --git a/ci/vfc_ci_report/plot.py b/ci/vfc_ci_report/plot.py new file mode 100644 index 0000000..f4bb20f --- /dev/null +++ b/ci/vfc_ci_report/plot.py @@ -0,0 +1,142 @@ +# General functions for filling plots with data in all report's views + +from bokeh.plotting import figure +from bokeh.models import HoverTool, TapTool, CustomJS + +from math import pi + + +def fill_dotplot( + plot, source, data_field, + tooltips=None, tooltips_formatters=None, + js_tap_callback=None, server_tap_callback=None, + lines=False, + lower_bound=False +): + + # (Optional) Tooltip and tooltip formatters + if tooltips is not None: + hover = HoverTool(tooltips=tooltips, mode="vline", names=["circle"]) + + if tooltips_formatters is not None: + hover.formatters = tooltips_formatters + + plot.add_tools(hover) + + # (Optional) Add TapTool (for JS tap callback) + if js_tap_callback is not None: + tap = TapTool(callback=CustomJS(code=js_tap_callback)) + plot.add_tools(tap) + + # (Optional) Add segment to represent a lower bound + if lower_bound: + lower_segment = plot.segment( + x0="%s_x" % data_field, y0=data_field, + x1="%s_x" % data_field, y1="%s_lower_bound" % data_field, + source=source, line_color="black" + ) + + # Draw dots (actually Bokeh circles) + circle = plot.circle( + name="circle", + x="%s_x" % data_field, y=data_field, source=source, size=12 + ) + + # (Optional) Draw lines between dots + if lines: + line = plot.line(x="%s_x" % data_field, y=data_field, source=source) + + # (Optional) Add server tap callback + if server_tap_callback is not None: + circle.data_source.selected.on_change("indices", server_tap_callback) + + # Plot appearance + plot.xgrid.grid_line_color = None + plot.ygrid.grid_line_color = None + + plot.yaxis[0].formatter.power_limit_high = 0 + plot.yaxis[0].formatter.power_limit_low = 0 + plot.yaxis[0].formatter.precision = 3 + + plot.xaxis[0].major_label_orientation = pi / 8 + + +def fill_boxplot( + plot, source, + prefix="", + tooltips=None, tooltips_formatters=None, + js_tap_callback=None, server_tap_callback=None +): + + # (Optional) Tooltip and tooltip formatters + if tooltips is not None: + hover = HoverTool(tooltips=tooltips, mode="vline", names=["full_box"]) + + if tooltips_formatters is not None: + hover.formatters = tooltips_formatters + + plot.add_tools(hover) + + # (Optional) Add TapTool (for JS tap callback) + if js_tap_callback is not None: + tap = TapTool(callback=CustomJS(code=js_tap_callback)) + plot.add_tools(tap) + + # Draw boxes (the prefix argument modifies the fields of ColumnDataSource + # that are used) + + if prefix != "": + prefix = "%s_" % prefix + + # Stems + top_stem = plot.segment( + x0="%sx" % prefix, y0="%smax" % prefix, + x1="%sx" % prefix, y1="%squantile75" % prefix, + source=source, line_color="black" + ) + bottom_stem = plot.segment( + x0="%sx" % prefix, y0="%smin" % prefix, + x1="%sx" % prefix, y1="%squantile25" % prefix, + source=source, line_color="black" + ) + + # Boxes + full_box = plot.vbar( + name="full_box", + x="%sx" % prefix, width=0.5, + top="%squantile75" % prefix, bottom="%squantile25" % prefix, + source=source, line_color="black" + ) + bottom_box = plot.vbar( + x="%sx" % prefix, width=0.5, + top="%squantile50" % prefix, bottom="%squantile25" % prefix, + source=source, line_color="black" + ) + + # Mu dot + mu_dot = plot.dot( + x="%sx" % prefix, y="%smu" % prefix, size=30, source=source, + color="black" + ) + + # (Optional) Add server tap callback + if server_tap_callback is not None: + top_stem.data_source.selected.on_change("indices", server_tap_callback) + bottom_stem.data_source.selected.on_change( + "indices", server_tap_callback) + + full_box.data_source.selected.on_change("indices", server_tap_callback) + bottom_box.data_source.selected.on_change( + "indices", server_tap_callback) + + mu_dot.data_source.selected.on_change("indices", server_tap_callback) + + # Plot appearance + plot.xgrid.grid_line_color = None + plot.ygrid.grid_line_color = None + + plot.yaxis[0].formatter.power_limit_high = 0 + plot.yaxis[0].formatter.power_limit_low = 0 + plot.yaxis[0].formatter.precision = 3 + + plot.xaxis[0].major_label_orientation = pi / 8 diff --git a/ci/vfc_ci_report/templates/index.html b/ci/vfc_ci_report/templates/index.html new file mode 100644 index 0000000..8fea8ec --- /dev/null +++ b/ci/vfc_ci_report/templates/index.html @@ -0,0 +1,482 @@ + + + + + Verificarlo Report + + + + + + + + + + {% extends base %} + + + + + {% block contents %} + + + + + + + +
+ + {% if has_logo %} + + {% else %} + + {% endif %} +
+ + + + + + + {% endblock %} + + + diff --git a/ci/workflow_templates/ci_README.j2.md b/ci/workflow_templates/ci_README.j2.md new file mode 100644 index 0000000..a41ca96 --- /dev/null +++ b/ci/workflow_templates/ci_README.j2.md @@ -0,0 +1,7 @@ +## Verificarlo CI : `{{dev_branch}}` + +You are on the `{{ci_branch}}` branch, which is automatically updated with the +[Verificarlo](https://github.com/verificarlo/verificarlo) test results from +`{{dev_branch}}` (in the `vfcruns` directory). + +You can start a Verificarlo CI server at anytime using the run files of this branch. diff --git a/ci/workflow_templates/gitlab-ci.j2.yml b/ci/workflow_templates/gitlab-ci.j2.yml new file mode 100644 index 0000000..de26aeb --- /dev/null +++ b/ci/workflow_templates/gitlab-ci.j2.yml @@ -0,0 +1,42 @@ +# This workflow will be executed when {{dev_branch}} is updated: +# it will run the configured tests and upload the results on vfc_ci_master. + +image: verificarlo/verificarlo + + +stages: + - run_verificarlo_tests + + +run_verificarlo_tests: + stage: run_verificarlo_tests + + before_script: + - git remote set-url origin https://{{username}}:${CI_PUSH_TOKEN}@{{remote_url}}.git + - git config --global user.email "{{email}}" + - git config --global user.name "{{username}}" + + script: + # We will probably drop these installations when integrating CI into + # Verificarlo + - pip install numpy scipy pandas bokeh jinja2 tables GitPython + - apt update + - apt install wget + - wget https://raw.githubusercontent.com/verificarlo/significantdigits/main/sigdigits.py -P /usr/local/lib/python3.8/dist-packages + + - ./vfc_ci test -g -r + - git_hash=$(git rev-parse --short "$CI_COMMIT_SHA") + - git fetch --all + - git checkout -b {{ci_branch}} origin/{{ci_branch}} + - mkdir -p vfcruns + - mv *.vfcrun.h5 vfcruns + - git add vfcruns/* + - git commit -m "[auto] New test results for commit ${git_hash}" + - git push + + rules: + - if: '$CI_COMMIT_BRANCH == "{{dev_branch}}"' + + artifacts: + paths: + - "*.vfcraw.h5" diff --git a/ci/workflow_templates/vfc_test_workflow.j2.yml b/ci/workflow_templates/vfc_test_workflow.j2.yml new file mode 100644 index 0000000..b39e21e --- /dev/null +++ b/ci/workflow_templates/vfc_test_workflow.j2.yml @@ -0,0 +1,57 @@ +# This workflow will be executed when {{dev_branch}} is updated: +# it will run the configured tests and upload the results on {{ci_branch}}. + +name: "Verificarlo CI ({{dev_branch}})" + +on: + # Triggers the workflow when {{dev_branch}} is updated + push: + branches: [ {{dev_branch}} ] + + workflow_dispatch: + + +jobs: + run_verificarlo_tests: + runs-on: ubuntu-latest + container: verificarlo/verificarlo + + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + + # We will probably drop these installations when integrating CI into + # Verificarlo + - name: Install Python requirements + run: | + pip install numpy scipy pandas bokeh jinja2 tables GitPython + apt update + apt install wget + wget https://raw.githubusercontent.com/verificarlo/significantdigits/main/sigdigits.py -P /usr/local/lib/python3.8/dist-packages + + - name: Run tests + # We assume the script is included in the repo for now + # (we'll probably want to remove "./" if the script ends up being integrated + # in Verificarlo and becomes available system-wide) + run: ./vfc_ci test -g -r + + - name: Commit test results + run: | + git_hash=$(git rev-parse --short "$GITHUB_SHA") + + git config --local user.email "action@github.com" + git config --local user.name "GitHub Action" + + git checkout {{ci_branch}} + mkdir -p vfcruns + mv *.vfcrun.h5 vfcruns + git add vfcruns/* + git commit -m "[auto] New test results for commit ${git_hash}" + git push + + - name: Upload raw results as artifacts + uses: actions/upload-artifact@v2 + with: + {% raw %}name: ${{github.sha}}.vfcraw{% endraw %} + path: ./*.vfcraw.h5 diff --git a/datasets/ci_dataset.hdf5 b/datasets/ci_dataset.hdf5 new file mode 100644 index 0000000..c7c91c2 Binary files /dev/null and b/datasets/ci_dataset.hdf5 differ diff --git a/include/vfc_hashmap.h b/include/vfc_hashmap.h new file mode 100644 index 0000000..c314f0e --- /dev/null +++ b/include/vfc_hashmap.h @@ -0,0 +1,248 @@ +/* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +/* + * Copyright 2012 Couchbase, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#define HASH_MULTIPLIER 31 +static const unsigned int hashmap_prime_1 = 73; +static const unsigned int hashmap_prime_2 = 5009; + +#ifndef __VFC_HASHMAP_HEADER__ + +struct vfc_hashmap_st { + size_t nbits; + size_t mask; + + size_t capacity; + size_t *items; + size_t nitems; + size_t n_deleted_items; +}; +typedef struct vfc_hashmap_st *vfc_hashmap_t; + +// allocate and initialize the map +vfc_hashmap_t vfc_hashmap_create(); + +// get the value at an index of a map +size_t get_value_at(size_t *items, size_t i); + +// get the key at an index of a map +size_t get_key_at(size_t *items, size_t i); + +// set the value at an index of a map +void set_value_at(size_t *items, size_t value, size_t i); + +// set the key at an index of a map +void set_key_at(size_t *items, size_t key, size_t i); + +// free the map +void vfc_hashmap_destroy(vfc_hashmap_t map); + +// insert an element in the map +void vfc_hashmap_insert(vfc_hashmap_t map, size_t key, void *item); + +// remove an element of the map +void vfc_hashmap_remove(vfc_hashmap_t map, size_t key); + +// test if an element is in the map +char vfc_hashmap_have(vfc_hashmap_t map, size_t key); + +// get an element of the map +void *vfc_hashmap_get(vfc_hashmap_t map, size_t key); + +// get the number of elements in the map +size_t vfc_hashmap_num_items(vfc_hashmap_t map); + +// Hash function +size_t vfc_hashmap_str_function(const char *id); + +#endif + +/***************** Verificarlo hashmap FUNCTIONS ******************** + * The following set of functions are used in backends and wrapper + * to stock and access quickly internal data. + *******************************************************************/ + +// free the map +void vfc_hashmap_destroy(vfc_hashmap_t map) { + if (map) { + free(map->items); + } + free(map); +} + +// allocate and initialize the map +vfc_hashmap_t vfc_hashmap_create() { + vfc_hashmap_t map = (vfc_hashmap_t) calloc(1, sizeof(struct vfc_hashmap_st)); + + if (map == NULL) { + return NULL; + } + map->nbits = 3; + map->capacity = (size_t)(1 << map->nbits); + map->mask = map->capacity - 1; + // an item is now a value and a key + map->items = (size_t *) calloc(map->capacity, 2 * sizeof(size_t)); + if (map->items == NULL) { + vfc_hashmap_destroy(map); + return NULL; + } + map->nitems = 0; + map->n_deleted_items = 0; + return map; +} + +size_t get_value_at(size_t *items, size_t i) { return items[i * 2]; } + +size_t get_key_at(size_t *items, size_t i) { return items[(i * 2) + 1]; } + +void set_value_at(size_t *items, size_t value, size_t i) { + items[i * 2] = value; +} + +void set_key_at(size_t *items, size_t key, size_t i) { + items[(i * 2) + 1] = key; +} + +// add a member in the table +static int hashmap_add_member(vfc_hashmap_t map, size_t key, void *item) { + size_t value = (size_t)item; + size_t ii; + + if (value == 0 || value == 1) { + return -1; + } + + ii = map->mask & (hashmap_prime_1 * key); + + while (get_value_at(map->items, ii) != 0 && + get_value_at(map->items, ii) != 1) { + if (get_value_at(map->items, ii) == value) { + return 0; + } else { + /* search free slot */ + ii = map->mask & (ii + hashmap_prime_2); + } + } + map->nitems++; + if (get_value_at(map->items, ii) == 1) { + map->n_deleted_items--; + } + + set_value_at(map->items, value, ii); + set_key_at(map->items, key, ii); + + return 1; +} + +// rehash the table if necessary +static void maybe_rehash_map(vfc_hashmap_t map) { + size_t *old_items; + size_t old_capacity, ii; + + if (map->nitems + map->n_deleted_items >= (double)map->capacity * 0.85) { + old_items = map->items; + old_capacity = map->capacity; + map->nbits++; + map->capacity = (size_t)(1 << map->nbits); + map->mask = map->capacity - 1; + map->items = (size_t *) calloc(map->capacity, 2 * sizeof(size_t)); + map->nitems = 0; + map->n_deleted_items = 0; + for (ii = 0; ii < old_capacity; ii++) { + hashmap_add_member(map, get_key_at(old_items, ii), + (void *)get_value_at(old_items, ii)); + } + free(old_items); + } +} + +// insert an element in the map +void vfc_hashmap_insert(vfc_hashmap_t map, size_t key, void *item) { + hashmap_add_member(map, key, item); + maybe_rehash_map(map); +} + +// remove an element of the map +void vfc_hashmap_remove(vfc_hashmap_t map, size_t key) { + size_t ii = map->mask & (hashmap_prime_1 * key); + + while (get_value_at(map->items, ii) != 0) { + if (get_key_at(map->items, ii) == key) { + set_value_at(map->items, 1, ii); + map->nitems--; + map->n_deleted_items++; + break; + } else { + ii = map->mask & (ii + hashmap_prime_2); + } + } +} + +// test if an element is in the map +char vfc_hashmap_have(vfc_hashmap_t map, size_t key) { + size_t ii = map->mask & (hashmap_prime_1 * key); + + while (get_value_at(map->items, ii) != 0) { + if (get_key_at(map->items, ii) == key) { + return 1; + } else { + ii = map->mask & (ii + hashmap_prime_2); + } + } + return 0; +} + +// get an element of the map +void *vfc_hashmap_get(vfc_hashmap_t map, size_t key) { + size_t ii = map->mask & (hashmap_prime_1 * key); + + while (get_value_at(map->items, ii) != 0) { + if (get_key_at(map->items, ii) == key) { + return (void *)get_value_at(map->items, ii); + } else { + ii = map->mask & (ii + hashmap_prime_2); + } + } + return NULL; +} + +// get the number of elements in the map +size_t vfc_hashmap_num_items(vfc_hashmap_t map) { return map->nitems; } + +// Hash function for strings +size_t vfc_hashmap_str_function(const char *id) { + unsigned const char *us; + + us = (unsigned const char *)id; + + size_t index = 0; + + while (*us != '\0') { + index = index * HASH_MULTIPLIER + *us; + us++; + } + + return index; +} + +// Free the hashmap +void vfc_hashmap_free(vfc_hashmap_t map) { + for (int ii = 0; ii < map->capacity; ii++) + if (get_value_at(map->items, ii) != 0 && get_value_at(map->items, ii) != 0) + free((void *)get_value_at(map->items, ii)); +} diff --git a/include/vfc_probe.h b/include/vfc_probe.h new file mode 100644 index 0000000..0d716d1 --- /dev/null +++ b/include/vfc_probe.h @@ -0,0 +1,254 @@ +/* +* This file defines "vfc_probes", a hashtable-based structure which can be used +* to place "probes" in a code and store the different values of test variables. +* These test results can then be exported in a CSV file, and used to generate a +* Verificarlo test report. +*/ + + +#include +#include +#include + +#include "vfc_hashmap.h" + +#define VAR_NAME(var) #var // Simply returns the name of var into a string + + +/* +* A probe containing a double value as well as its key, which is needed when +* dumping the probes +*/ + +struct vfc_probe_node { + char * key; + double value; +}; + +typedef struct vfc_probe_node vfc_probe_node; + + + +/* +* The probes structure. It simply acts as a wrapper for a Verificarlo hashmap. +*/ + +struct vfc_probes { + vfc_hashmap_t map; +}; + +typedef struct vfc_probes vfc_probes; + + +/* +* Initialize an empty vfc_probes instance +*/ + +vfc_probes vfc_init_probes() { + vfc_probes probes; + probes.map = vfc_hashmap_create(); + + return probes; +} + + + +/* +* Free all probes +*/ + +void vfc_free_probes(vfc_probes * probes) { + + // Before freeing the map, iterate manually over all items to free the keys + vfc_probe_node * probe = NULL; + for(int i = 0; i < probes->map->capacity; i++) { + probe = (vfc_probe_node*) get_value_at(probes->map->items, i); + if(probe != NULL) { + if(probe->key != NULL) { + free(probe->key); + } + } + } + + vfc_hashmap_free(probes->map); +} + + + +/* +* Helper function to generate the key from test and variable name +*/ + +char * gen_probe_key(char * testName, char * varName) { + char * key = (char *) malloc(strlen(testName) + strlen(varName) + 2); + strcpy(key, testName); + strcat(key, ","); + strcat(key, varName); + + return key; +} + + + +/* +* Helper function to detect forbidden character ',' in the keys +*/ + +void validate_probe_key(char * str) { + unsigned int len = strlen(str); + + for(unsigned int i=0; imap, vfc_hashmap_str_function(key) + ); + + if(oldProbe != NULL) { + if(strcmp(key, oldProbe->key) == 0) { + fprintf( + stderr, + "Error [verificarlo]: you have a duplicate error with one of \ + your probes (\"%s\"). Please make sure to use different names.\n", + key + ); + exit(1); + } + } + + // Insert the element in the hashmap + vfc_probe_node * newProbe = (vfc_probe_node*) malloc(sizeof(vfc_probe_node)); + newProbe->key = key; + newProbe->value = val; + + vfc_hashmap_insert( + probes->map, vfc_hashmap_str_function(key), newProbe + ); + + return 0; +} + + + +/* +* Remove (free) an element from the hash table +*/ + +int vfc_remove_probe(vfc_probes * probes, char * testName, char * varName) { + + if(probes == NULL) { + return 1; + } + + // Get the key, which is : testName + "," + varName + char * key = gen_probe_key(testName, varName); + + vfc_hashmap_remove(probes->map, vfc_hashmap_str_function(key)); + + return 0; +} + + + +/* +* Return the number of probes stored in the hashmap +*/ + +unsigned int vfc_num_probes(vfc_probes * probes) { + return vfc_hashmap_num_items(probes->map); +} + + + +/* +* Dump probes in a .csv file (the double values are converted to hex), then +* free it. +*/ + +int vfc_dump_probes(vfc_probes * probes) { + + if(probes == NULL) { + return 1; + } + + // Get export path from the VFC_PROBES_OUTPUT env variable + char* exportPath = getenv("VFC_PROBES_OUTPUT"); + if(!exportPath) { + printf( + "Warning [verificarlo]: VFC_PROBES_OUTPUT is not set, probes will \ + not be dumped\n" + ); + vfc_free_probes(probes); + return 0; + } + + FILE * fp = fopen(exportPath, "w"); + + if(fp == NULL) { + fprintf( + stderr, + "Error [verificarlo]: impossible to open the CSV file to save your \ + probes (\"%s\")\n", + exportPath + ); + exit(1); + } + + // First line gives the column names + fprintf(fp, "test,variable,value\n"); + + // Iterate over all table elements + vfc_probe_node * probe = NULL; + for(int i = 0; i < probes->map->capacity; i++) { + probe = (vfc_probe_node*) get_value_at(probes->map->items, i); + if(probe != NULL) { + fprintf( + fp, "%s,%a\n", + probe->key, + probe->value + ); + } + } + + fclose(fp); + + vfc_free_probes(probes); + + return 0; +} diff --git a/tests/vfc_test_h5.cpp b/tests/vfc_test_h5.cpp new file mode 100644 index 0000000..52b2bb6 --- /dev/null +++ b/tests/vfc_test_h5.cpp @@ -0,0 +1,202 @@ +// This files is almost the same as test_h5.cpp, with the difference that it +// dumps Verificarlo probes for vfc_ci integration, and that it reads a list of +// cycles in a CSV file, instead of accepting a start and an end cycle (which +// makes it easier to select the exact cycles we are interested in with vfc_ci). + +#include +#include + +#include +#include +#include + + +#include "SM_MaponiA3.hpp" +#include "SM_Standard.hpp" +#include "SM_Helpers.hpp" +#include "vfc_probe.h" + +using namespace H5; +// #define DEBUG + +const H5std_string FILE_NAME( "datasets/ci_dataset.hdf5" ); + +double residual_max(double * A, unsigned int Dim) { + double max = 0.0; + for (unsigned int i = 0; i < Dim; i++) { + for (unsigned int j = 0; j < Dim; j++) { + double delta = (A[i * Dim + j] - (i == j)); + delta = abs(delta); + if (delta > max) max = delta; + } + } + return max; +} + +double residual2(double * A, unsigned int Dim) { + double res = 0.0; + for (unsigned int i = 0; i < Dim; i++) { + for (unsigned int j = 0; j < Dim; j++) { + double delta = (A[i * Dim + j] - (i == j)); + res += delta*delta; + } + } + return res; +} + +void read_int(H5File file, std::string key, unsigned int * data) { + DataSet ds = file.openDataSet(key); + ds.read(data, PredType::STD_U32LE); + ds.close(); +} + +void read_double(H5File file, std::string key, double * data) { + DataSet ds = file.openDataSet(key); + ds.read(data, PredType::IEEE_F64LE); + ds.close(); +} + + +/* Return a vector containing all cycles to execute by reading a data file */ +std::vector get_cycles_list(std::string path) { + std::ifstream file_stream(path); + std::stringstream string_stream; + string_stream << file_stream.rdbuf(); + + std::string cycle_str; + std::vector cycles_list = {}; + + while(string_stream >> cycle_str) { + cycles_list.push_back(std::stoi(cycle_str)); + } + + return cycles_list; +} + +int test_cycle(H5File file, int cycle, std::string version, vfc_probes * probes) { + + /* Read the data */ + + std::string group = "cycle_" + std::to_string(cycle); + + // This will result in the same string as group but with the cycle number + // being zero-padded. This is used when calling vfc_put_probe later on. + std::string zero_padded_group = std::to_string(cycle); + zero_padded_group = "cycle_" + + std::string(5 - zero_padded_group.length(), '0') + zero_padded_group; + + try{ + file.openGroup(group); + } catch(H5::Exception& e){ + std::cerr << "group " << group << "not found" << std::endl; + return 0; + } + + unsigned int dim, nupdates, col, i, j; + read_int(file, group + "/slater_matrix_dim", &dim); + read_int(file, group + "/nupdates", &nupdates); + + + double * slater_matrix = new double[dim*dim]; + read_double(file, group + "/slater_matrix", slater_matrix); + + double * slater_inverse = new double[dim*dim]; + read_double(file, group + "/slater_inverse", slater_inverse); + //slater_inverse = transpose(slater_inverse, dim); + + unsigned int * col_update_index = new unsigned int[nupdates]; + read_int(file, group + "/col_update_index", col_update_index); + + double * updates = new double[nupdates*dim]; + read_double(file, group + "/updates", updates); + + double * u = new double[nupdates*dim]; + + /* Test */ +#ifdef DEBUG + showMatrix(slater_matrix, dim, "OLD Slater"); +#endif + +#ifdef DEBUG + showMatrix(slater_inverse, dim, "OLD Inverse"); +#endif + + for (j = 0; j < nupdates; j++) { + for (i = 0; i < dim; i++) { + col = col_update_index[j]; + u[i + j*dim] = updates[i + j*dim] - slater_matrix[i*dim + (col - 1)]; + slater_matrix[i*dim + (col - 1)] = updates[i + j*dim]; + } + } + + if (version == "maponia3") { + MaponiA3(slater_inverse, dim, nupdates, u, col_update_index); + } else if (version == "sm1") { + SM1(slater_inverse, dim, nupdates, u, col_update_index); + } else if (version == "sm2") { + SM2(slater_inverse, dim, nupdates, u, col_update_index); + } else if (version == "sm3") { + SM3(slater_inverse, dim, nupdates, u, col_update_index); + } else { + std::cerr << "Unknown version " << version << std::endl; + exit(1); + } + +#ifdef DEBUG + showMatrix(slater_matrix, dim, "NEW Slater"); +#endif + +#ifdef DEBUG + showMatrix(slater_inverse, dim, "NEW Inverse"); +#endif + + double * res = new double[dim*dim] {0}; + matMul(slater_matrix, slater_inverse, res, dim); + bool ok = is_identity(res, dim, 1e-3); + + double res_max = residual_max(res, dim); + double res2 = residual2(res, dim); + +#ifdef DEBUG + showMatrix(res, dim, "Result"); +#endif + + vfc_put_probe(probes, &(zero_padded_group)[0], &("res_max_" + version)[0], res_max); + vfc_put_probe(probes, &(zero_padded_group)[0], &("res2_" + version)[0], res2); + + delete [] res, updates, u, col_update_index, + slater_matrix, slater_inverse; + + return ok; +} + +int main(int argc, char **argv) { + if (argc != 3) { + std::cerr << "Execute from within '/'" << std::endl; + std::cerr << "usage: test_h5 " << std::endl; + return 1; + } + std::string version(argv[1]); + std::vector cycles_list = get_cycles_list(argv[2]); + H5File file(FILE_NAME, H5F_ACC_RDONLY); + + vfc_probes probes = vfc_init_probes(); + probes = vfc_init_probes(); + + bool ok; + for (int i = 0; i < cycles_list.size(); i++) { + ok = test_cycle(file, cycles_list[i], version, &probes); + if (ok) { + std::cout << "ok -- cycle " << std::to_string(i) + << std::endl; + } + else { + std::cerr << "failed -- cycle " << std::to_string(i) + << std::endl; + } + } + + vfc_dump_probes(&probes); + + return ok; +} diff --git a/tools/extract-from-h5.py b/tools/extract-from-h5.py new file mode 100644 index 0000000..6907be6 --- /dev/null +++ b/tools/extract-from-h5.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python +import h5py +import sys + +# Helper script to extract a few cycles from a large dataset. This will be +# especially useful for the Verificarlo CI, since vfc_ci_cycles.txt can be +# used to both extract (with this script), and read the small dataset (in a CI +# run). + + + # Parse arguments + +if len(sys.argv) != 4: + sys.stderr.write( + "Error: Wrong number of arguments. Usage : extract_h5.py "\ + " \n" + ) + exit(1) + +source_dataset_path = sys.argv[1] +cycles_list_path = sys.argv[2] +destination_dataset_path = sys.argv[3] + + + # Read the cycles list + +cycles_list = [] + +try: + f = open(cycles_list_path) + for line in f: + cycles_list.extend([cycle for cycle in line.split()]) +except IOError: + sys.stderr.write("Error: Could not read " + cycles_list_path + "\n") + exit(1) +finally: + f.close() + + + # Read the source dataset, and extract the cycles to the destination dataset + +try: + fs = h5py.File(source_dataset_path, "r") + +except IOError: + sys.stderr.write("Error: Could not read " + source_dataset_path + "\n") + exit(1) + +fd = h5py.File(destination_dataset_path, "w") + +# Copy cycles groups + +groups = [ + "slater_matrix_dim", + "nupdates", + "slater_matrix", + "slater_inverse", + "col_update_index", + "updates" +] + +for cycle in cycles_list: + cycle_name = "cycle_" + cycle + + new_cycle = fd.create_group(cycle_name) + + # Copy all datasets + for group_name in groups: + fs.copy(cycle_name + "/" + group_name, new_cycle) + + +print("Dataset successfully exported to %s" % source_dataset_path) diff --git a/vfc_ci b/vfc_ci new file mode 100755 index 0000000..be0afdb --- /dev/null +++ b/vfc_ci @@ -0,0 +1,199 @@ +#!/usr/bin/env python3 + +# This is the entry point of the Verificarlo CI command line interface, which is +# based on argparse and this article : +# https://mike.depalatis.net/blog/simplifying-argparse.html +# From here, 3 subcommands can be called : +# - setup : create a vfc_ci branch and workflow on the current Git repo +# - test : run and export test results according to the vfc_tests_config.json +# - serve : launch a Bokeh server to visualize run results + +import argparse + +########################################################################## + +# Parameters validation helpers + + +def is_port(string): + value = int(string) + if value < 0 or value > 65535: + raise argparse.ArgumentTypeError("Value has to be between 0 and 65535") + return value + + +def is_directory(string): + import os + + isdir = os.path.isdir(string) + if not isdir: + raise argparse.ArgumentTypeError("Directory does not exist") + + return string + + +########################################################################## + + # Subcommand decorator + +cli = argparse.ArgumentParser( + description="Define, run, automatize, and visualize custom Verificarlo tests." +) +subparsers = cli.add_subparsers(dest="subcommand") + + +def subcommand(description="", args=[], parent=subparsers): + def decorator(func): + parser = parent.add_parser(func.__name__, description=description) + for arg in args: + parser.add_argument(*arg[0], **arg[1]) + parser.set_defaults(func=func) + return decorator + + +def argument(*name_or_flags, **kwargs): + return ([*name_or_flags], kwargs) + + +########################################################################## + + # "setup" subcommand + +@subcommand( + description="Create an automated workflow to execute Verificarlo tests.", + args=[ + argument( + "git_host", + help=""" + specify where your repository is hosted + """, + choices=["github", "gitlab"] + ) + ] +) +def setup(args): + import ci.setup + ci.setup.setup(args.git_host) + + # "test" subcommand + + +@subcommand( + description="Execute predefined Verificarlo tests and save their results.", + args=[ + argument( + "-g", "--is-git-commit", + help=""" + When specified, the last Git commit of the local repository (working + directory) will be fetched and associated with the run. + """, + action="store_true" + ), + argument( + "-r", "--export-raw-results", + help=""" + Specify if an additional HDF5 file containing the raw results must be + exported. + """, + action="store_true" + ), + argument( + "-d", "--dry-run", + help=""" + Perform a dry run by not saving the test results. + """, + action="store_true" + ) + ] +) +def test(args): + import ci.test + ci.test.run(args.is_git_commit, args.export_raw_results, args.dry_run) + + # "serve" subcommand + + +@subcommand( + description=""" + Start a server to visualize Verificarlo test results. + """, + args=[ + argument( + "-s", "--show", + help=""" + Specify if the report must be opened in the browser at server + startup. + """, + action="store_true" + ), + argument( + "-gd", "--git-directory", + help=""" + Path to a local Git repository. The report will be linked to the + remote URL (GitHub and GitLab are supported). + """, + type=is_directory + ), + argument( + "-gu", "--git-url", + help=""" + GitHub or GitLab repository URL. The report will be linked to this + URL. + """, + type=str + ), + argument( + "-p", "--port", + help=""" + The port on which the server will run. Defaults to 8080. + """, + type=is_port, + default=8080 + ), + argument( + "-a", "--allow-origin", + help=""" + The origin (URL) from which the report will be accessible. + Port number must not be specified. Defaults to * (allow everything). + """, + type=str, + default="*" + ), + argument( + "-l", "--logo", + help=""" + Specify the URL of an image to be displayed in the report header. + """, + type=str + ) + ] +) +def serve(args): + + # git_directory and git_url are supposed to be exclusive + if args.git_directory is not None and args.git_url is not None: + raise argparse.ArgumentTypeError( + "\"-gd\" / \"--git-directory\" and \"-gu\" / \"--git-url\" are " + "mutually exclusive. Please make sure to use at most one of them." + ) + + import ci.serve + ci.serve.serve( + args.show, + args.git_directory, + args.git_url, + args.port, + args.allow_origin, + args.logo + ) + + +############################################################################### + + # Main command group and entry point +if __name__ == "__main__": + args = cli.parse_args() + if args.subcommand is None: + cli.print_help() + else: + args.func(args) diff --git a/vfc_ci_cycles.txt b/vfc_ci_cycles.txt new file mode 100644 index 0000000..840505a --- /dev/null +++ b/vfc_ci_cycles.txt @@ -0,0 +1,2 @@ +1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 +327 655 983 1311 1639 1967 2295 2623 2951 3279 3607 3935 4263 4591 4919 5247 5575 5903 6231 6559 6887 7215 7543 7871 8199 diff --git a/vfc_tests_config.json b/vfc_tests_config.json new file mode 100644 index 0000000..d036855 --- /dev/null +++ b/vfc_tests_config.json @@ -0,0 +1,92 @@ +{ + "make_command": "make -f Makefile.verificarlo", + "executables": [ + { + "executable": "bin/vfc_test_h5", + "parameters" : "maponia3 vfc_ci_cycles.txt", + "vfc_backends": [ + { + "name": "libinterflop_mca.so", + "repetitions": 50 + } + ] + }, + + { + "executable": "bin/vfc_test_h5", + "parameters" : "sm1 vfc_ci_cycles.txt", + "vfc_backends": [ + { + "name": "libinterflop_mca.so", + "repetitions": 50 + } + ] + }, + + { + "executable": "bin/vfc_test_h5", + "parameters" : "sm2 vfc_ci_cycles.txt", + "vfc_backends": [ + { + "name": "libinterflop_mca.so", + "repetitions": 50 + } + ] + }, + + { + "executable": "bin/vfc_test_h5", + "parameters" : "sm3 vfc_ci_cycles.txt", + "vfc_backends": [ + { + "name": "libinterflop_mca.so", + "repetitions": 50 + } + ] + }, + + { + "executable": "bin/vfc_test_h5", + "parameters" : "maponia3 vfc_ci_cycles.txt", + "vfc_backends": [ + { + "name": "libinterflop_mca.so --mode=rr", + "repetitions": 50 + } + ] + }, + + { + "executable": "bin/vfc_test_h5", + "parameters" : "sm1 vfc_ci_cycles.txt", + "vfc_backends": [ + { + "name": "libinterflop_mca.so --mode=rr", + "repetitions": 50 + } + ] + }, + + { + "executable": "bin/vfc_test_h5", + "parameters" : "sm2 vfc_ci_cycles.txt", + "vfc_backends": [ + { + "name": "libinterflop_mca.so --mode=rr", + "repetitions": 50 + } + ] + }, + + { + "executable": "bin/vfc_test_h5", + "parameters" : "sm3 vfc_ci_cycles.txt", + "vfc_backends": [ + { + "name": "libinterflop_mca.so --mode=rr", + "repetitions": 50 + } + ] + } + ] +}