From 0114fd7b143abaaf8b3347ecfe15ed5ed6099136 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Delval?= Date: Thu, 17 Jun 2021 16:50:42 +0200 Subject: [PATCH] Remove source code of vfc_ci and improve vfc_test_h5.cpp Since vfc_ci has been merged wih the main verificarlo repository and is now available in Verificarlo's Docker, the source code of vfc_ci can now be removed. Also, vfc_test_h5.cpp has been modified so that every algorithm is called directly. This results in a much shorter vfc_tests_config.json since we don't need to vall vfc_test_h5 with all algorithms as arguments. --- .github/workflows/vfc_test_workflow.yml | 15 +- Makefile.verificarlo | 2 +- ci/__init__.py | 0 ci/serve.py | 28 - ci/setup.py | 146 ----- ci/test.py | 367 ----------- ci/vfc_ci_report/compare_runs.py | 542 ----------------- ci/vfc_ci_report/helper.py | 170 ------ ci/vfc_ci_report/inspect_runs.py | 570 ------------------ ci/vfc_ci_report/main.py | 208 ------- ci/vfc_ci_report/plot.py | 142 ----- ci/vfc_ci_report/templates/index.html | 482 --------------- ci/workflow_templates/ci_README.j2.md | 7 - ci/workflow_templates/gitlab-ci.j2.yml | 42 -- .../vfc_test_workflow.j2.yml | 57 -- tests/vfc_test_h5.cpp | 25 +- vfc_ci | 199 ------ vfc_tests_config.json | 63 +- 18 files changed, 21 insertions(+), 3044 deletions(-) delete mode 100644 ci/__init__.py delete mode 100755 ci/serve.py delete mode 100644 ci/setup.py delete mode 100755 ci/test.py delete mode 100644 ci/vfc_ci_report/compare_runs.py delete mode 100644 ci/vfc_ci_report/helper.py delete mode 100644 ci/vfc_ci_report/inspect_runs.py delete mode 100644 ci/vfc_ci_report/main.py delete mode 100644 ci/vfc_ci_report/plot.py delete mode 100644 ci/vfc_ci_report/templates/index.html delete mode 100644 ci/workflow_templates/ci_README.j2.md delete mode 100644 ci/workflow_templates/gitlab-ci.j2.yml delete mode 100644 ci/workflow_templates/vfc_test_workflow.j2.yml delete mode 100755 vfc_ci diff --git a/.github/workflows/vfc_test_workflow.yml b/.github/workflows/vfc_test_workflow.yml index 61bad51..edc585c 100644 --- a/.github/workflows/vfc_test_workflow.yml +++ b/.github/workflows/vfc_test_workflow.yml @@ -21,35 +21,24 @@ jobs: with: fetch-depth: 0 - # We will probably drop these installations when integrating CI into - # Verificarlo - - name: Install Python requirements + - name: Install HDF5 requirements run: | - pip install numpy scipy pandas bokeh jinja2 tables GitPython apt update apt install -y wget libhdf5-dev g++ - wget https://raw.githubusercontent.com/verificarlo/significantdigits/main/sigdigits.py -P /usr/local/lib/python3.8/dist-packages - - name: Run tests - # We assume the script is included in the repo for now - # (we'll probably want to remove "./" if the script ends up being integrated - # in Verificarlo and becomes available system-wide) - run: ./vfc_ci test -g -r + run: vfc_ci test -g -r - name: Commit test results run: | git_hash=$(git rev-parse --short "$GITHUB_SHA") - git config --local user.email "action@github.com" git config --local user.name "GitHub Action" - git checkout vfc_ci_dev mkdir -p vfcruns mv *.vfcrun.h5 vfcruns git add vfcruns/* git commit -m "[auto] New test results for commit ${git_hash}" git push - - name: Upload raw results as artifacts uses: actions/upload-artifact@v2 with: diff --git a/Makefile.verificarlo b/Makefile.verificarlo index b4b5603..b093482 100644 --- a/Makefile.verificarlo +++ b/Makefile.verificarlo @@ -20,7 +20,7 @@ ifeq ($(MKL),-DMKL) ifeq ($(ENV),INTEL) LFLAGS = -mkl=sequential # implicit else - LFLAGS = $(H5LFLAGS) + LFLAGS = $(H5LFLAGS) -lvfc_probes endif endif H5CXXFLAGS = $(CXXFLAGS) diff --git a/ci/__init__.py b/ci/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/ci/serve.py b/ci/serve.py deleted file mode 100755 index 37d91f2..0000000 --- a/ci/serve.py +++ /dev/null @@ -1,28 +0,0 @@ -# Server for the Verificarlo CI report. This is simply a wrapper to avoid -# calling Bokeh directly. - -import os - - -def serve(show, git_directory, git_url, port, allow_origin, logo_url): - - # Prepare arguments - show = "--show" if show else "" - - git = "" - if git_directory is not None: - git = "git directory %s" % git_directory - if git_url is not None: - git = "git url %s" % git_url - - logo = "" - if logo_url is not None: - logo = "logo %s" % logo_url - - dirname = os.path.dirname(__file__) - - # Call the "bokeh serve" command on the system - command = "bokeh serve %s/vfc_ci_report %s --allow-websocket-origin=%s:%s --port %s --args %s %s" \ - % (dirname, show, allow_origin, port, port, git, logo) - - os.system(command) diff --git a/ci/setup.py b/ci/setup.py deleted file mode 100644 index 1bd50b3..0000000 --- a/ci/setup.py +++ /dev/null @@ -1,146 +0,0 @@ -# Helper script to set up Verificarlo CI on the current branch - -import git - -import sys -import os -from jinja2 import Environment, FileSystemLoader - -########################################################################## - -# Helper functions - - -def gen_readme(dev_branch, ci_branch): - - # Init template loader - path = os.path.dirname(os.path.abspath(__file__)) - env = Environment(loader=FileSystemLoader(path)) - template = env.get_template("workflow_templates/ci_README.j2.md") - - # Render template - render = template.render(dev_branch=dev_branch, ci_branch=ci_branch) - - # Write template - with open("README.md", "w") as fh: - fh.write(render) - - -def gen_workflow(git_host, dev_branch, ci_branch, repo): - - # Init template loader - path = os.path.dirname(os.path.abspath(__file__)) - env = Environment(loader=FileSystemLoader(path)) - - if git_host == "github": - # Load template - template = env.get_template( - "workflow_templates/vfc_test_workflow.j2.yml") - - # Render it - render = template.render(dev_branch=dev_branch, ci_branch=ci_branch) - - # Write the file - filename = ".github/workflows/vfc_test_workflow.yml" - os.makedirs(os.path.dirname(filename), exist_ok=True) - with open(filename, "w") as fh: - fh.write(render) - - if git_host == "gitlab": - template = env.get_template("workflow_templates/gitlab-ci.j2.yml") - - # Ask for the user who will run the jobs (Gitlab specific) - username = input( - "[vfc_ci] Enter the name of the user who will run the CI jobs:") - email = input( - "[vfc_ci] Enter the e-mail of the user who will run the CI jobs:") - - remote_url = repo.remotes[0].config_reader.get("url") - remote_url = remote_url.replace("http://", "") - remote_url = remote_url.replace("https://", "") - - render = template.render( - dev_branch=dev_branch, - ci_branch=ci_branch, - username=username, - email=email, - remote_url=remote_url - ) - - filename = ".gitlab-ci.yml" - with open(filename, "w") as fh: - fh.write(render) - - -########################################################################## - -def setup(git_host): - - # Init repo and make sure that the workflow setup is possible - - repo = git.Repo(".") - repo.remotes.origin.fetch() - - # Make sure that repository is clean - assert(not repo.is_dirty()), "Error [vfc_ci]: Unstaged changes detected " \ - "in your work tree." - - dev_branch = repo.active_branch - dev_branch_name = str(dev_branch) - dev_remote = dev_branch.tracking_branch() - - # Make sure that the active branch (on which to setup the workflow) has a - # remote - assert(dev_remote is not None), "Error [vfc_ci]: The current branch doesn't " \ - "have a remote." - - # Make sure that we are not behind the remote (so we can push safely later) - rev = "%s...%s" % (dev_branch_name, str(dev_remote)) - commits_behind = list(repo.iter_commits(rev)) - assert(commits_behind == []), "Error [vfc_ci]: The local branch seems " \ - "to be at least one commit behind remote." - - # Commit the workflow on the current (dev) branch - - ci_branch_name = "vfc_ci_%s" % dev_branch_name - gen_workflow(git_host, dev_branch_name, ci_branch_name, repo) - repo.git.add(".") - repo.index.commit("[auto] Set up Verificarlo CI on this branch") - repo.remote(name="origin").push() - - # Create the CI branch (orphan branch with a readme on it) - # (see : https://github.com/gitpython-developers/GitPython/issues/615) - - repo.head.reference = git.Head(repo, "refs/heads/" + ci_branch_name) - - repo.index.remove(["*"]) - gen_readme(dev_branch_name, ci_branch_name) - repo.index.add(["README.md"]) - - repo.index.commit( - "[auto] Create the Verificarlo CI branch for %s" % dev_branch_name, - parent_commits=None - ) - repo.remote(name="origin").push( - refspec="%s:%s" % (ci_branch_name, ci_branch_name) - ) - - # Force checkout back to the original (dev) branch - repo.git.checkout(dev_branch_name, force=True) - - # Print termination messages - - print( - "Info [vfc_ci]: A Verificarlo CI workflow has been setup on " - "%s." % dev_branch_name - ) - print( - "Info [vfc_ci]: Make sure that you have a \"vfc_tests_config.json\" on " - "this branch. You can also perform a \"vfc_ci test\" dry run before " - "pushing other commits.") - - if git_host == "gitlab": - print( - "Info [vfc_ci]: Since you are using GitLab, make sure that you " - "have created an access token for the user you specified (registered " - "as a variable called \"CI_PUSH_TOKEN\" in your repository).") diff --git a/ci/test.py b/ci/test.py deleted file mode 100755 index bc38714..0000000 --- a/ci/test.py +++ /dev/null @@ -1,367 +0,0 @@ -# This script reads the vfc_tests_config.json file and executes tests accordingly -# It will also generate a ... .vfcrunh5 file with the results of the run - -import sigdigits as sd -import scipy.stats -import numpy as np -import pandas as pd -import os -import json - -import calendar -import time - -# Forcing an older pickle protocol allows backwards compatibility when reading -# HDF5 written in 3.8+ using an older version of Python -import pickle -pickle.HIGHEST_PROTOCOL = 4 - - -# Magic numbers -min_pvalue = 0.05 -max_zscore = 3 - - -########################################################################## - -# Helper functions - -# Read a CSV file outputted by vfc_probe as a Pandas dataframe -def read_probes_csv(filepath, backend, warnings, execution_data): - - try: - results = pd.read_csv(filepath) - - except FileNotFoundError: - print( - "Warning [vfc_ci]: Probes not found, your code might have crashed " - "or you might have forgotten to call vfc_dump_probes" - ) - warnings.append(execution_data) - return pd.DataFrame( - columns=["test", "variable", "values", "vfc_backend"] - ) - - except Exception: - print( - "Warning [vfc_ci]: Your probes could not be read for some unknown " - "reason" - ) - warnings.append(execution_data) - return pd.DataFrame( - columns=["test", "variable", "values", "vfc_backend"] - ) - - if len(results) == 0: - print( - "Warning [vfc_ci]: Probes empty, it looks like you have dumped " - "them without calling vfc_put_probe" - ) - warnings.append(execution_data) - - # Once the CSV has been opened and validated, return its content - results["value"] = results["value"].apply(lambda x: float.fromhex(x)) - results.rename(columns={"value": "values"}, inplace=True) - - results["vfc_backend"] = backend - - return results - - -# Wrappers to sd.significant_digits (returns results in base 2) - -def significant_digits(x): - - # If the null hypothesis is rejected, call sigdigits with the General - # formula: - if x.pvalue < min_pvalue: - # In a pandas DF, "values" actually refers to the array of columns, and - # not the column named "values" - distribution = x.values[3] - distribution = distribution.reshape(len(distribution), 1) - - # The distribution's empirical average will be used as the reference - mu = np.array([x.mu]) - - s = sd.significant_digits( - distribution, - mu, - precision=sd.Precision.Relative, - method=sd.Method.General, - - probability=0.9, - confidence=0.95 - ) - - # s is returned inside a list - return s[0] - - # Else, manually compute sMCA (Stott-Parker formula) - else: - return -np.log2(np.absolute(x.sigma / x.mu)) - - -def significant_digits_lower_bound(x): - # If the null hypothesis is rejected, no lower bound - if x.pvalue < min_pvalue: - return x.s2 - - # Else, the lower bound will be computed with p= .9 alpha-1=.95 - else: - distribution = x.values[3] - distribution = distribution.reshape(len(distribution), 1) - - mu = np.array([x.mu]) - - s = sd.significant_digits( - distribution, - mu, - precision=sd.Precision.Relative, - method=sd.Method.CNH, - - probability=0.9, - confidence=0.95 - ) - - return s[0] - - -########################################################################## - - # Main functions - - -# Open and read the tests config file -def read_config(): - try: - with open("vfc_tests_config.json", "r") as file: - data = file.read() - - except FileNotFoundError as e: - e.strerror = "Error [vfc_ci]: This file is required to describe the tests "\ - "to run and generate a Verificarlo run file" - raise e - - return json.loads(data) - - -# Set up metadata -def generate_metadata(is_git_commit): - - # Metadata and filename are initiated as if no commit was associated - metadata = { - "timestamp": calendar.timegm(time.gmtime()), - "is_git_commit": is_git_commit, - "hash": "", - "author": "", - "message": "" - } - - if is_git_commit: - print("Fetching metadata from last commit...") - from git import Repo - - repo = Repo(".") - head_commit = repo.head.commit - - metadata["timestamp"] = head_commit.authored_date - - metadata["hash"] = str(head_commit)[0:7] - metadata["author"] = "%s <%s>" \ - % (str(head_commit.author), head_commit.author.email) - metadata["message"] = head_commit.message.split("\n")[0] - - return metadata - - -# Execute tests and collect results in a Pandas dataframe (+ dataprocessing) -def run_tests(config): - - # Run the build command - print("Info [vfc_ci]: Building tests...") - os.system(config["make_command"]) - - # This is an array of Pandas dataframes for now - data = [] - - # Create tmp folder to export results - os.system("mkdir .vfcruns.tmp") - n_files = 0 - - # This will contain all executables/repetition numbers from which we could - # not get any data - warnings = [] - - # Tests execution loop - for executable in config["executables"]: - print("Info [vfc_ci]: Running executable :", - executable["executable"], "...") - - parameters = "" - if "parameters" in executable: - parameters = executable["parameters"] - - for backend in executable["vfc_backends"]: - - export_backend = "VFC_BACKENDS=\"" + backend["name"] + "\" " - command = "./" + executable["executable"] + " " + parameters - - repetitions = 1 - if "repetitions" in backend: - repetitions = backend["repetitions"] - - # Run test repetitions and save results - for i in range(repetitions): - file = ".vfcruns.tmp/%s.csv" % str(n_files) - export_output = "VFC_PROBES_OUTPUT=\"%s\" " % file - os.system(export_output + export_backend + command) - - # This will only be used if we need to append this exec to the - # warnings list - execution_data = { - "executable": executable["executable"], - "backend": backend["name"], - "repetition": i + 1 - } - - data.append(read_probes_csv( - file, - backend["name"], - warnings, - execution_data - )) - - n_files = n_files + 1 - - # Clean CSV output files (by deleting the tmp folder) - os.system("rm -rf .vfcruns.tmp") - - # Combine all separate executions in one dataframe - data = pd.concat(data, sort=False, ignore_index=True) - data = data.groupby(["test", "vfc_backend", "variable"])\ - .values.apply(list).reset_index() - - # Make sure we have some data to work on - assert(len(data) != 0), "Error [vfc_ci]: No data have been generated " \ - "by your tests executions, aborting run without writing results file" - - return data, warnings - - # Data processing - - -def data_processing(data): - - data["values"] = data["values"].apply(lambda x: np.array(x).astype(float)) - - # Get empirical average, standard deviation and p-value - data["mu"] = data["values"].apply(np.average) - data["sigma"] = data["values"].apply(np.std) - data["pvalue"] = data["values"].apply( - lambda x: scipy.stats.shapiro(x).pvalue) - - # Significant digits - data["s2"] = data.apply(significant_digits, axis=1) - data["s10"] = data["s2"].apply(lambda x: sd.change_base(x, 10)) - - # Lower bound of the confidence interval using the sigdigits module - data["s2_lower_bound"] = data.apply(significant_digits_lower_bound, axis=1) - data["s10_lower_bound"] = data["s2_lower_bound"].apply( - lambda x: sd.change_base(x, 10)) - - # Compute moments of the distribution - # (including a new distribution obtained by filtering outliers) - data["values"] = data["values"].apply(np.sort) - - data["mu"] = data["values"].apply(np.average) - data["min"] = data["values"].apply(np.min) - data["quantile25"] = data["values"].apply(np.quantile, args=(0.25,)) - data["quantile50"] = data["values"].apply(np.quantile, args=(0.50,)) - data["quantile75"] = data["values"].apply(np.quantile, args=(0.75,)) - data["max"] = data["values"].apply(np.max) - - data["nsamples"] = data["values"].apply(len) - - # Display all executions that resulted in a warning - - -def show_warnings(warnings): - if len(warnings) > 0: - print( - "Warning [vfc_ci]: Some of your runs could not generate any data " - "(for instance because your code crashed) and resulted in " - "warnings. Here is the complete list :" - ) - - for i in range(0, len(warnings)): - print("- Warning %s:" % i) - - print(" Executable: %s" % warnings[i]["executable"]) - print(" Backend: %s" % warnings[i]["backend"]) - print(" Repetition: %s" % warnings[i]["repetition"]) - - -########################################################################## - - # Entry point - -def run(is_git_commit, export_raw_values, dry_run): - - # Get config, metadata and data - print("Info [vfc_ci]: Reading tests config file...") - config = read_config() - - print("Info [vfc_ci]: Generating run metadata...") - metadata = generate_metadata(is_git_commit) - - data, warnings = run_tests(config) - show_warnings(warnings) - - # Data processing - print("Info [vfc_ci]: Processing data...") - data_processing(data) - - # Prepare data for export (by creating a proper index and linking run - # timestamp) - data = data.set_index(["test", "variable", "vfc_backend"]).sort_index() - data["timestamp"] = metadata["timestamp"] - - filename = metadata["hash"] if is_git_commit else str( - metadata["timestamp"]) - - # Prepare metadata for export - metadata = pd.DataFrame.from_dict([metadata]) - metadata = metadata.set_index("timestamp") - - # NOTE : Exporting to HDF5 requires to install "tables" on the system - - # Export raw data if needed - if export_raw_values and not dry_run: - data.to_hdf(filename + ".vfcraw.h5", key="data") - metadata.to_hdf(filename + ".vfcraw.h5", key="metadata") - - # Export data - del data["values"] - if not dry_run: - data.to_hdf(filename + ".vfcrun.h5", key="data") - metadata.to_hdf(filename + ".vfcrun.h5", key="metadata") - - # Print termination messages - print( - "Info [vfc_ci]: The results have been successfully written to " - "%s.vfcrun.h5." - % filename - ) - - if export_raw_values: - print( - "Info [vfc_ci]: A file containing the raw values has also been " - "created : %s.vfcraw.h5." - % filename - ) - - if dry_run: - print( - "Info [vfc_ci]: The dry run flag was enabled, so no files were " - "actually created." - ) diff --git a/ci/vfc_ci_report/compare_runs.py b/ci/vfc_ci_report/compare_runs.py deleted file mode 100644 index d05b676..0000000 --- a/ci/vfc_ci_report/compare_runs.py +++ /dev/null @@ -1,542 +0,0 @@ -# Manage the view comparing a variable over different runs - -import time - -import pandas as pd - -from math import pi - -from bokeh.plotting import figure, curdoc -from bokeh.embed import components -from bokeh.models import Select, ColumnDataSource, Panel, Tabs, HoverTool, \ - TextInput, CheckboxGroup, TapTool, CustomJS - -import helper -import plot - - -########################################################################## - - -class CompareRuns: - - # Helper functions related to CompareRuns - - # From an array of timestamps, returns the array of runs names (for the x - # axis ticks), as well as the metadata (in a dict of arrays) associated to - # this array (for the tooltips) - def gen_x_series(self, timestamps): - - # Initialize the objects to return - x_series = [] - x_metadata = dict( - date=[], - is_git_commit=[], - hash=[], - author=[], - message=[] - ) - - # n == 0 means we want all runs, we also make sure not to go out of - # bound if asked for more runs than we have - n = self.current_n_runs - if n == 0 or n > len(timestamps): - n = len(timestamps) - - for i in range(0, n): - # Get metadata associated to this run - row_metadata = helper.get_metadata( - self.metadata, timestamps[-i - 1]) - date = time.ctime(timestamps[-i - 1]) - - # Fill the x series - str = row_metadata["name"] - x_series.insert(0, helper.get_metadata( - self.metadata, timestamps[-i - 1])["name"]) - - # Fill the metadata lists - x_metadata["date"].insert(0, date) - x_metadata["is_git_commit"].insert( - 0, row_metadata["is_git_commit"]) - x_metadata["hash"].insert(0, row_metadata["hash"]) - x_metadata["author"].insert(0, row_metadata["author"]) - x_metadata["message"].insert(0, row_metadata["message"]) - - return x_series, x_metadata - - # Plots update function - - def update_plots(self): - - # Select all data matching current test/var/backend - - runs = self.data.loc[[self.widgets["select_test"].value], - self.widgets["select_var"].value, - self.widgets["select_backend"].value] - - timestamps = runs["timestamp"] - x_series, x_metadata = self.gen_x_series(timestamps.sort_values()) - - # Update source - - main_dict = runs.to_dict("series") - main_dict["x"] = x_series - - # Add metadata (for tooltip) - main_dict.update(x_metadata) - - # Select the last n runs only - n = self.current_n_runs - main_dict = {key: value[-n:] for key, value in main_dict.items()} - - # Generate ColumnDataSources for the 3 dotplots - for stat in ["sigma", "s10", "s2"]: - dict = { - "%s_x" % stat: main_dict["x"], - - "is_git_commit": main_dict["is_git_commit"], - "date": main_dict["date"], - "hash": main_dict["hash"], - "author": main_dict["author"], - "message": main_dict["message"], - - stat: main_dict[stat], - - "nsamples": main_dict["nsamples"], - } - - if stat == "s10" or stat == "s2": - dict["%s_lower_bound" % - stat] = main_dict["%s_lower_bound" % - stat] - - # Filter outliers if the box is checked - if len(self.widgets["outliers_filtering_compare"].active) > 0: - outliers = helper.detect_outliers(dict[stat]) - dict[stat] = helper.remove_outliers(dict[stat], outliers) - dict["%s_x" % stat] = helper.remove_outliers( - dict["%s_x" % stat], outliers) - - # Assign ColumnDataSource - self.sources["%s_source" % stat].data = dict - - # Generate ColumnDataSource for the boxplot - dict = { - "is_git_commit": main_dict["is_git_commit"], - "date": main_dict["date"], - "hash": main_dict["hash"], - "author": main_dict["author"], - "message": main_dict["message"], - - "x": main_dict["x"], - "min": main_dict["min"], - "quantile25": main_dict["quantile25"], - "quantile50": main_dict["quantile50"], - "quantile75": main_dict["quantile75"], - "max": main_dict["max"], - "mu": main_dict["mu"], - "pvalue": main_dict["pvalue"], - - "nsamples": main_dict["nsamples"] - } - - self.sources["boxplot_source"].data = dict - - # Update x axis - - helper.reset_x_range( - self.plots["boxplot"], - self.sources["boxplot_source"].data["x"] - ) - helper.reset_x_range( - self.plots["sigma_plot"], - self.sources["sigma_source"].data["sigma_x"] - ) - helper.reset_x_range( - self.plots["s10_plot"], - self.sources["s10_source"].data["s10_x"] - ) - helper.reset_x_range( - self.plots["s2_plot"], - self.sources["s2_source"].data["s2_x"] - ) - - # Widgets' callback functions - - def update_test(self, attrname, old, new): - - # If the value is updated by the CustomJS, self.widgets["select_var"].value - # won't be updated, so we have to look for that case and assign it - # manually - - # "new" should be a list when updated by CustomJS - if isinstance(new, list): - # If filtering removed all options, we might have an empty list - # (in this case, we just skip the callback and do nothing) - if len(new) > 0: - new = new[0] - else: - return - - if new != self.widgets["select_test"].value: - # The callback will be triggered again with the updated value - self.widgets["select_test"].value = new - return - - # New list of available vars - self.vars = self.data.loc[new]\ - .index.get_level_values("variable").drop_duplicates().tolist() - self.widgets["select_var"].options = self.vars - - # Reset var selection if old one is not available in new vars - if self.widgets["select_var"].value not in self.vars: - self.widgets["select_var"].value = self.vars[0] - # The update_var callback will be triggered by the assignment - - else: - # Trigger the callback manually (since the plots need to be updated - # anyway) - self.update_var("", "", self.widgets["select_var"].value) - - def update_var(self, attrname, old, new): - - # If the value is updated by the CustomJS, self.widgets["select_var"].value - # won't be updated, so we have to look for that case and assign it - # manually - - # new should be a list when updated by CustomJS - if isinstance(new, list): - new = new[0] - - if new != self.widgets["select_var"].value: - # The callback will be triggered again with the updated value - self.widgets["select_var"].value = new - return - - # New list of available backends - self.backends = self.data.loc[self.widgets["select_test"].value, self.widgets["select_var"].value]\ - .index.get_level_values("vfc_backend").drop_duplicates().tolist() - self.widgets["select_backend"].options = self.backends - - # Reset backend selection if old one is not available in new backends - if self.widgets["select_backend"].value not in self.backends: - self.widgets["select_backend"].value = self.backends[0] - # The update_backend callback will be triggered by the assignment - - else: - # Trigger the callback manually (since the plots need to be updated - # anyway) - self.update_backend("", "", self.widgets["select_backend"].value) - - def update_backend(self, attrname, old, new): - - # Simply update plots, since no other data is affected - self.update_plots() - - def update_n_runs(self, attrname, old, new): - # Simply update runs selection (value and string display) - self.widgets["select_n_runs"].value = new - self.current_n_runs = self.n_runs_dict[self.widgets["select_n_runs"].value] - - self.update_plots() - - def update_outliers_filtering(self, attrname, old, new): - self.update_plots() - - # Bokeh setup functions - - def setup_plots(self): - - tools = "pan, wheel_zoom, xwheel_zoom, ywheel_zoom, reset, save" - - # Custom JS callback that will be used when tapping on a run - # Only switches the view, a server callback is required to update plots - # (defined inside template to avoid bloating server w/ too much JS code) - js_tap_callback = "goToInspectRuns();" - - # Box plot - self.plots["boxplot"] = figure( - name="boxplot", title="Variable distribution over runs", - plot_width=900, plot_height=400, x_range=[""], - tools=tools, sizing_mode="scale_width" - ) - - box_tooltips = [ - ("Git commit", "@is_git_commit"), - ("Date", "@date"), - ("Hash", "@hash"), - ("Author", "@author"), - ("Message", "@message"), - ("Min", "@min{%0.18e}"), - ("Max", "@max{%0.18e}"), - ("1st quartile", "@quantile25{%0.18e}"), - ("Median", "@quantile50{%0.18e}"), - ("3rd quartile", "@quantile75{%0.18e}"), - ("μ", "@mu{%0.18e}"), - ("p-value", "@pvalue"), - ("Number of samples", "@nsamples") - ] - box_tooltips_formatters = { - "@min": "printf", - "@max": "printf", - "@quantile25": "printf", - "@quantile50": "printf", - "@quantile75": "printf", - "@mu": "printf" - } - - plot.fill_boxplot( - self.plots["boxplot"], self.sources["boxplot_source"], - tooltips=box_tooltips, - tooltips_formatters=box_tooltips_formatters, - js_tap_callback=js_tap_callback, - server_tap_callback=self.inspect_run_callback_boxplot, - ) - self.doc.add_root(self.plots["boxplot"]) - - # Sigma plot (bar plot) - self.plots["sigma_plot"] = figure( - name="sigma_plot", title="Standard deviation σ over runs", - plot_width=900, plot_height=400, x_range=[""], - tools=tools, sizing_mode="scale_width" - ) - - sigma_tooltips = [ - ("Git commit", "@is_git_commit"), - ("Date", "@date"), - ("Hash", "@hash"), - ("Author", "@author"), - ("Message", "@message"), - ("σ", "@sigma"), - ("Number of samples", "@nsamples") - ] - - plot.fill_dotplot( - self.plots["sigma_plot"], self.sources["sigma_source"], "sigma", - tooltips=sigma_tooltips, - js_tap_callback=js_tap_callback, - server_tap_callback=self.inspect_run_callback_sigma, - lines=True - ) - self.doc.add_root(self.plots["sigma_plot"]) - - # s plot (bar plot with 2 tabs) - self.plots["s10_plot"] = figure( - name="s10_plot", title="Significant digits s over runs", - plot_width=900, plot_height=400, x_range=[""], - tools=tools, sizing_mode="scale_width" - ) - - s10_tooltips = [ - ("Git commit", "@is_git_commit"), - ("Date", "@date"), - ("Hash", "@hash"), - ("Author", "@author"), - ("Message", "@message"), - ("s", "@s10"), - ("s lower bound", "@s10_lower_bound"), - ("Number of samples", "@nsamples") - ] - - plot.fill_dotplot( - self.plots["s10_plot"], self.sources["s10_source"], "s10", - tooltips=s10_tooltips, - js_tap_callback=js_tap_callback, - server_tap_callback=self.inspect_run_callback_s10, - lines=True, - lower_bound=True - ) - s10_tab = Panel(child=self.plots["s10_plot"], title="Base 10") - - self.plots["s2_plot"] = figure( - name="s2_plot", title="Significant digits s over runs", - plot_width=900, plot_height=400, x_range=[""], - tools=tools, sizing_mode="scale_width" - ) - - s2_tooltips = [ - ("Git commit", "@is_git_commit"), - ("Date", "@date"), - ("Hash", "@hash"), - ("Author", "@author"), - ("Message", "@message"), - ("s", "@s2"), - ("s lower bound", "@s2_lower_bound"), - ("Number of samples", "@nsamples") - ] - - plot.fill_dotplot( - self.plots["s2_plot"], self.sources["s2_source"], "s2", - tooltips=s2_tooltips, - js_tap_callback=js_tap_callback, - server_tap_callback=self.inspect_run_callback_s2, - lines=True, - lower_bound=True - ) - s2_tab = Panel(child=self.plots["s2_plot"], title="Base 2") - - s_tabs = Tabs( - name="s_tabs", - tabs=[s10_tab, s2_tab], - tabs_location="below" - ) - - self.doc.add_root(s_tabs) - - def setup_widgets(self): - - # Initial selections - - # Test/var/backend combination (we select all first elements at init) - self.tests = self.data\ - .index.get_level_values("test").drop_duplicates().tolist() - - self.vars = self.data.loc[self.tests[0]]\ - .index.get_level_values("variable").drop_duplicates().tolist() - - self.backends = self.data.loc[self.tests[0], self.vars[0]]\ - .index.get_level_values("vfc_backend").drop_duplicates().tolist() - - # Custom JS callback that will be used client side to filter selections - filter_callback_js = """ - selector.options = options.filter(e => e.includes(cb_obj.value)); - """ - - # Test selector widget - - # Number of runs to display - # The dict structure allows us to get int value from the display string - # in O(1) - self.n_runs_dict = { - "Last 3 runs": 3, - "Last 5 runs": 5, - "Last 10 runs": 10, - "All runs": 0 - } - - # Contains all options strings - n_runs_display = list(self.n_runs_dict.keys()) - - # Will be used when updating plots (contains actual number to diplay) - self.current_n_runs = self.n_runs_dict[n_runs_display[1]] - - # Selector widget - self.widgets["select_test"] = Select( - name="select_test", title="Test :", - value=self.tests[0], options=self.tests - ) - self.doc.add_root(self.widgets["select_test"]) - self.widgets["select_test"].on_change("value", self.update_test) - self.widgets["select_test"].on_change("options", self.update_test) - - # Filter widget - self.widgets["test_filter"] = TextInput( - name="test_filter", title="Tests filter:" - ) - self.widgets["test_filter"].js_on_change( - "value", - CustomJS( - args=dict( - options=self.tests, - selector=self.widgets["select_test"]), - code=filter_callback_js)) - self.doc.add_root(self.widgets["test_filter"]) - - # Number of runs to display - - self.widgets["select_n_runs"] = Select( - name="select_n_runs", title="Display :", - value=n_runs_display[1], options=n_runs_display - ) - self.doc.add_root(self.widgets["select_n_runs"]) - self.widgets["select_n_runs"].on_change("value", self.update_n_runs) - - # Variable selector widget - - self.widgets["select_var"] = Select( - name="select_var", title="Variable :", - value=self.vars[0], options=self.vars - ) - self.doc.add_root(self.widgets["select_var"]) - self.widgets["select_var"].on_change("value", self.update_var) - self.widgets["select_var"].on_change("options", self.update_var) - - # Backend selector widget - - self.widgets["select_backend"] = Select( - name="select_backend", title="Verificarlo backend :", - value=self.backends[0], options=self.backends - ) - self.doc.add_root(self.widgets["select_backend"]) - self.widgets["select_backend"].on_change("value", self.update_backend) - - # Outliers filtering checkbox - - self.widgets["outliers_filtering_compare"] = CheckboxGroup( - name="outliers_filtering_compare", - labels=["Filter outliers"], active=[] - ) - self.doc.add_root(self.widgets["outliers_filtering_compare"]) - self.widgets["outliers_filtering_compare"]\ - .on_change("active", self.update_outliers_filtering) - - # Communication methods - # (to send/receive messages to/from master) - - # Callback to change view of Inspect runs when data is selected - - def inspect_run_callback(self, new, source_name, x_name): - - # In case we just unselected everything, then do nothing - if new == []: - return - - index = new[-1] - run_name = self.sources[source_name].data[x_name][index] - - self.master.go_to_inspect(run_name) - - # Wrappers for each plot (since new is the index of the clicked element, - # it is dependent of the plot because we could have filtered some outliers) - # There doesn't seem to be an easy way to add custom parameters to a - # Bokeh callback, so using wrappers seems to be the best solution for now - - def inspect_run_callback_boxplot(self, attr, old, new): - self.inspect_run_callback(new, "boxplot_source", "x") - - def inspect_run_callback_sigma(self, attr, old, new): - self.inspect_run_callback(new, "sigma_source", "sigma_x") - - def inspect_run_callback_s2(self, attr, old, new): - self.inspect_run_callback(new, "s2_source", "s2_x") - - def inspect_run_callback_s10(self, attr, old, new): - self.inspect_run_callback(new, "s10_source", "s10_x") - - # Constructor - - def __init__(self, master, doc, data, metadata): - - self.master = master - - self.doc = doc - self.data = data - self.metadata = metadata - - self.sources = { - "boxplot_source": ColumnDataSource(data={}), - "sigma_source": ColumnDataSource(data={}), - "s10_source": ColumnDataSource(data={}), - "s2_source": ColumnDataSource(data={}) - } - - self.plots = {} - self.widgets = {} - - # Setup Bokeh objects - self.setup_plots() - self.setup_widgets() - - # At this point, everything should have been initialized, so we can - # show the plots for the first time - self.update_plots() diff --git a/ci/vfc_ci_report/helper.py b/ci/vfc_ci_report/helper.py deleted file mode 100644 index f258959..0000000 --- a/ci/vfc_ci_report/helper.py +++ /dev/null @@ -1,170 +0,0 @@ -# General helper functions for both compare_runs and compare_variables - -import calendar -import time -from itertools import compress - -import numpy as np - -# Magic numbers -max_ticks = 15 -max_zscore = 3 - -########################################################################## - - -# From a timestamp, return the associated metadata as a Pandas serie -def get_metadata(metadata, timestamp): - return metadata.loc[timestamp] - - -# Convert a metadata Pandas series to a JS readable dict -def metadata_to_dict(metadata): - dict = metadata.to_dict() - - # JS doesn't accept True for booleans, and Python doesn't accept true - # (because of the caps) => using an integer is a portable solution - dict["is_git_commit"] = 1 if dict["is_git_commit"] else 0 - - dict["date"] = time.ctime(metadata.name) - - return dict - - -# Return a string that indicates the elapsed time since the run, used as the -# x-axis tick in "Compare runs" or when selecting run in "Inspect run" -def get_run_name(timestamp, hash): - - gmt = time.gmtime() - now = calendar.timegm(gmt) - diff = now - timestamp - - # Special case : < 1 minute (return string directly) - if diff < 60: - str = "Less than a minute ago" - - if hash != "": - str = str + " (%s)" % hash - - if str == get_run_name.previous: - get_run_name.counter = get_run_name.counter + 1 - str = "%s (%s)" % (str, get_run_name.counter) - else: - get_run_name.counter = 0 - get_run_name.previous = str - - return str - - # < 1 hour - if diff < 3600: - n = int(diff / 60) - str = "%s minute%s ago" - # < 1 day - elif diff < 86400: - n = int(diff / 3600) - str = "%s hour%s ago" - # < 1 week - elif diff < 604800: - n = int(diff / 86400) - str = "%s day%s ago" - # < 1 month - elif diff < 2592000: - n = int(diff / 604800) - str = "%s week%s ago" - # > 1 month - else: - n = diff / 2592000 - str = "%s month%s ago" - - plural = "" - if n != 1: - plural = "s" - - str = str % (n, plural) - - # We might want to add the git hash - if hash != "": - str = str + " (%s)" % hash - - # Finally, check for duplicate with previously generated string - if str == get_run_name.previous: - # Increment the duplicate counter and add it to str - get_run_name.counter = get_run_name.counter + 1 - str = "%s (%s)" % (str, get_run_name.counter) - - else: - # No duplicate, reset both previously generated str and duplicate - # counter - get_run_name.counter = 0 - get_run_name.previous = str - - return str - - -# These external variables will store data about the last generated string to -# avoid duplicates (assuming the runs are sorted by time) -get_run_name.counter = 0 -get_run_name.previous = "" - - -def reset_run_strings(): - get_run_name.counter = 0 - get_run_name.previous = "" - - -# Update all the x-ranges from a dict of plots -def reset_x_range(plot, x_range): - plot.x_range.factors = x_range - - if len(x_range) < max_ticks: - plot.xaxis.major_tick_line_color = "#000000" - plot.xaxis.minor_tick_line_color = "#000000" - - plot.xaxis.major_label_text_font_size = "8pt" - - else: - plot.xaxis.major_tick_line_color = None - plot.xaxis.minor_tick_line_color = None - - plot.xaxis.major_label_text_font_size = "0pt" - - -# Return an array of booleans that indicate which elements are outliers -# (True means element is not an outlier and must be kept) -def detect_outliers(array, max_zscore=max_zscore): - if len(array) <= 2: - return [True] * len(array) - - median = np.median(array) - std = np.std(array) - if std == 0: - return array - distance = abs(array - median) - # Array of booleans with elements to be filtered - outliers_array = distance < max_zscore * std - - return outliers_array - - -def remove_outliers(array, outliers): - return list(compress(array, outliers)) - - -def remove_boxplot_outliers(dict, outliers, prefix): - outliers = detect_outliers(dict["%s_max" % prefix]) - - dict["%s_x" % prefix] = remove_outliers(dict["%s_x" % prefix], outliers) - - dict["%s_min" % prefix] = remove_outliers( - dict["%s_min" % prefix], outliers) - dict["%s_quantile25" % prefix] = remove_outliers( - dict["%s_quantile25" % prefix], outliers) - dict["%s_quantile50" % prefix] = remove_outliers( - dict["%s_quantile50" % prefix], outliers) - dict["%s_quantile75" % prefix] = remove_outliers( - dict["%s_quantile75" % prefix], outliers) - dict["%s_max" % prefix] = remove_outliers( - dict["%s_max" % prefix], outliers) - dict["%s_mu" % prefix] = remove_outliers(dict["%s_mu" % prefix], outliers) - - dict["nsamples"] = remove_outliers(dict["nsamples"], outliers) diff --git a/ci/vfc_ci_report/inspect_runs.py b/ci/vfc_ci_report/inspect_runs.py deleted file mode 100644 index f53ab17..0000000 --- a/ci/vfc_ci_report/inspect_runs.py +++ /dev/null @@ -1,570 +0,0 @@ -# Manage the view comparing the variables of a run - -from math import pi -from functools import partial - -import pandas as pd -import numpy as np - -from bokeh.plotting import figure, curdoc -from bokeh.embed import components -from bokeh.models import Select, ColumnDataSource, Panel, Tabs, HoverTool,\ - RadioButtonGroup, CheckboxGroup, CustomJS - -import helper -import plot - - -########################################################################## - - -class InspectRuns: - - # Helper functions related to InspectRun - - # Returns a dictionary mapping user-readable strings to all run timestamps - def gen_runs_selection(self): - - runs_dict = {} - - # Iterate over timestamp rows (runs) and fill dict - for row in self.metadata.iloc: - # The syntax used by pandas makes this part a bit tricky : - # row.name is the index of metadata (so it refers to the - # timestamp), whereas row["name"] is the column called "name" - # (which is the display string used for the run) - - # runs_dict[run's name] = run's timestamp - runs_dict[row["name"]] = row.name - - return runs_dict - - def gen_boxplot_tooltips(self, prefix): - return [ - ("Name", "@%s_x" % prefix), - ("Min", "@" + prefix + "_min{%0.18e}"), - ("Max", "@" + prefix + "_max{%0.18e}"), - ("1st quartile", "@" + prefix + "_quantile25{%0.18e}"), - ("Median", "@" + prefix + "_quantile50{%0.18e}"), - ("3rd quartile", "@" + prefix + "_quantile75{%0.18e}"), - ("μ", "@" + prefix + "_mu{%0.18e}"), - ("Number of samples (tests)", "@nsamples") - ] - - def gen_boxplot_tooltips_formatters(self, prefix): - return { - "@%s_min" % prefix: "printf", - "@%s_max" % prefix: "printf", - "@%s_quantile25" % prefix: "printf", - "@%s_quantile50" % prefix: "printf", - "@%s_quantile75" % prefix: "printf", - "@%s_mu" % prefix: "printf" - } - - # Data processing helper - # (computes new distributions for sigma, s2, s10) - - def data_processing(self, dataframe): - - # Compute aggragated mu - dataframe["mu"] = np.vectorize( - np.average)( - dataframe["mu"], - weights=dataframe["nsamples"]) - - # nsamples is the number of aggregated elements (as well as the number - # of samples for our new sigma and s distributions) - dataframe["nsamples"] = dataframe["nsamples"].apply(lambda x: len(x)) - - dataframe["mu_x"] = dataframe.index - # Make sure that strings don't excede a certain length - dataframe["mu_x"] = dataframe["mu_x"].apply( - lambda x: x[:17] + "[...]" + x[-17:] if len(x) > 39 else x - ) - - # Get quantiles and mu for sigma, s10, s2 - for prefix in ["sigma", "s10", "s2"]: - - dataframe["%s_x" % prefix] = dataframe["mu_x"] - - dataframe[prefix] = dataframe[prefix].apply(np.sort) - - dataframe["%s_min" % prefix] = dataframe[prefix].apply(np.min) - dataframe["%s_quantile25" % prefix] = dataframe[prefix].apply( - np.quantile, args=(0.25,)) - dataframe["%s_quantile50" % prefix] = dataframe[prefix].apply( - np.quantile, args=(0.50,)) - dataframe["%s_quantile75" % prefix] = dataframe[prefix].apply( - np.quantile, args=(0.75,)) - dataframe["%s_max" % prefix] = dataframe[prefix].apply(np.max) - dataframe["%s_mu" % prefix] = dataframe[prefix].apply(np.average) - del dataframe[prefix] - - return dataframe - - # Plots update function - - def update_plots(self): - - groupby_display = self.widgets["groupby_radio"].labels[ - self.widgets["groupby_radio"].active - ] - groupby = self.factors_dict[groupby_display] - - filterby_display = self.widgets["filterby_radio"].labels[ - self.widgets["filterby_radio"].active - ] - filterby = self.factors_dict[filterby_display] - - # Groupby and aggregate lines belonging to the same group in lists - - groups = self.run_data[ - self.run_data.index.isin( - [self.widgets["select_filter"].value], - level=filterby - ) - ].groupby(groupby) - - groups = groups.agg({ - "sigma": lambda x: x.tolist(), - "s10": lambda x: x.tolist(), - "s2": lambda x: x.tolist(), - - "mu": lambda x: x.tolist(), - - # Used for mu weighted average first, then will be replaced - "nsamples": lambda x: x.tolist() - }) - - # Compute the new distributions, ... - groups = self.data_processing(groups).to_dict("list") - - # Update source - - # Assign each ColumnDataSource, starting with the boxplots - for prefix in ["sigma", "s10", "s2"]: - - dict = { - "%s_x" % prefix: groups["%s_x" % prefix], - "%s_min" % prefix: groups["%s_min" % prefix], - "%s_quantile25" % prefix: groups["%s_quantile25" % prefix], - "%s_quantile50" % prefix: groups["%s_quantile50" % prefix], - "%s_quantile75" % prefix: groups["%s_quantile75" % prefix], - "%s_max" % prefix: groups["%s_max" % prefix], - "%s_mu" % prefix: groups["%s_mu" % prefix], - - "nsamples": groups["nsamples"] - } - - # Filter outliers if the box is checked - if len(self.widgets["outliers_filtering_inspect"].active) > 0: - - # Boxplots will be filtered by max then min - top_outliers = helper.detect_outliers(dict["%s_max" % prefix]) - helper.remove_boxplot_outliers(dict, top_outliers, prefix) - - bottom_outliers = helper.detect_outliers( - dict["%s_min" % prefix]) - helper.remove_boxplot_outliers(dict, bottom_outliers, prefix) - - self.sources["%s_source" % prefix].data = dict - - # Finish with the mu plot - dict = { - "mu_x": groups["mu_x"], - "mu": groups["mu"], - - "nsamples": groups["nsamples"] - } - - self.sources["mu_source"].data = dict - - # Filter outliers if the box is checked - if len(self.widgets["outliers_filtering_inspect"].active) > 0: - mu_outliers = helper.detect_outliers(groups["mu"]) - groups["mu"] = helper.remove_outliers(groups["mu"], mu_outliers) - groups["mu_x"] = helper.remove_outliers( - groups["mu_x"], mu_outliers) - - # Update plots axis/titles - - # Get display string of the last (unselected) factor - factors_dict = self.factors_dict.copy() - del factors_dict[groupby_display] - del factors_dict[filterby_display] - for_all = list(factors_dict.keys())[0] - - # Update all display strings for plot title (remove caps, plural) - groupby_display = groupby_display.lower() - filterby_display = filterby_display.lower()[:-1] - for_all = for_all.lower() - - self.plots["mu_inspect"].title.text = \ - "Empirical average μ of %s (groupped by %s, for all %s)" \ - % (filterby_display, groupby_display, for_all) - - self.plots["sigma_inspect"].title.text = \ - "Standard deviation σ of %s (groupped by %s, for all %s)" \ - % (filterby_display, groupby_display, for_all) - - self.plots["s10_inspect"].title.text = \ - "Significant digits s of %s (groupped by %s, for all %s)" \ - % (filterby_display, groupby_display, for_all) - - self.plots["s2_inspect"].title.text = \ - "Significant digits s of %s (groupped by %s, for all %s)" \ - % (filterby_display, groupby_display, for_all) - - helper.reset_x_range(self.plots["mu_inspect"], groups["mu_x"]) - helper.reset_x_range(self.plots["sigma_inspect"], groups["sigma_x"]) - helper.reset_x_range(self.plots["s10_inspect"], groups["s10_x"]) - helper.reset_x_range(self.plots["s2_inspect"], groups["s2_x"]) - - # Widets' callback functions - - # Run selector callback - - def update_run(self, attrname, old, new): - - filterby = self.widgets["filterby_radio"].labels[ - self.widgets["filterby_radio"].active - ] - filterby = self.factors_dict[filterby] - - # Update run selection (by using dict mapping) - self.current_run = self.runs_dict[new] - - # Update run data - self.run_data = self.data[self.data["timestamp"] == self.current_run] - - # Save old selected option - old_value = self.widgets["select_filter"].value - - # Update filter options - options = self.run_data.index\ - .get_level_values(filterby).drop_duplicates().tolist() - self.widgets["select_filter"].options = options - - if old_value not in self.widgets["select_filter"].options: - self.widgets["select_filter"].value = options[0] - # The update_var callback will be triggered by the assignment - - else: - # Trigger the callback manually (since the plots need to be updated - # anyway) - self.update_filter("", "", old_value) - - # "Group by" radio - - def update_groupby(self, attrname, old, new): - - # Update "Filter by" radio list - filterby_list = list(self.factors_dict.keys()) - del filterby_list[self.widgets["groupby_radio"].active] - self.widgets["filterby_radio"].labels = filterby_list - - filterby = self.widgets["filterby_radio"].labels[ - self.widgets["filterby_radio"].active - ] - filterby = self.factors_dict[filterby] - - # Save old selected option - old_value = self.widgets["select_filter"].value - - # Update filter options - options = self.run_data.index\ - .get_level_values(filterby).drop_duplicates().tolist() - self.widgets["select_filter"].options = options - - if old_value not in self.widgets["select_filter"].options: - self.widgets["select_filter"].value = options[0] - # The update_var callback will be triggered by the assignment - - else: - # Trigger the callback manually (since the plots need to be updated - # anyway) - self.update_filter("", "", old_value) - - # "Filter by" radio - - def update_filterby(self, attrname, old, new): - - filterby = self.widgets["filterby_radio"].labels[ - self.widgets["filterby_radio"].active - ] - filterby = self.factors_dict[filterby] - - # Save old selected option - old_value = self.widgets["select_filter"].value - - # Update filter selector options - options = self.run_data.index\ - .get_level_values(filterby).drop_duplicates().tolist() - self.widgets["select_filter"].options = options - - if old_value not in self.widgets["select_filter"].options: - self.widgets["select_filter"].value = options[0] - # The update_var callback will be triggered by the assignment - - else: - # Trigger the callback manually (since the plots need to be updated - # anyway) - self.update_filter("", "", old_value) - - # Filter selector callback - - def update_filter(self, attrname, old, new): - self.update_plots() - - # Filter outliers checkbox callback - - def update_outliers_filtering(self, attrname, old, new): - # The status (checked/unchecked) of the checkbox is also verified inside - # self.update_plots(), so calling this function is enough - self.update_plots() - - # Bokeh setup functions - # (for both variable and backend selection at once) - - def setup_plots(self): - - tools = "pan, wheel_zoom, xwheel_zoom, ywheel_zoom, reset, save" - - # Tooltips and formatters - - dotplot_tooltips = [ - ("Name", "@mu_x"), - ("μ", "@mu{%0.18e}"), - ("Number of samples (tests)", "@nsamples") - ] - dotplot_formatters = { - "@mu": "printf" - } - - sigma_boxplot_tooltips = self.gen_boxplot_tooltips("sigma") - sigma_boxplot_tooltips_formatters = self.gen_boxplot_tooltips_formatters( - "sigma") - - s10_boxplot_tooltips = self.gen_boxplot_tooltips("s10") - s10_boxplot_tooltips_formatters = self.gen_boxplot_tooltips_formatters( - "s10") - - s2_boxplot_tooltips = self.gen_boxplot_tooltips("s2") - s2_boxplot_tooltips_formatters = self.gen_boxplot_tooltips_formatters( - "s2") - - # Plots - - # Mu plot - self.plots["mu_inspect"] = figure( - name="mu_inspect", - title="", - plot_width=900, plot_height=400, x_range=[""], - tools=tools, sizing_mode="scale_width" - ) - plot.fill_dotplot( - self.plots["mu_inspect"], self.sources["mu_source"], "mu", - tooltips=dotplot_tooltips, - tooltips_formatters=dotplot_formatters - ) - self.doc.add_root(self.plots["mu_inspect"]) - - # Sigma plot - self.plots["sigma_inspect"] = figure( - name="sigma_inspect", - title="", - plot_width=900, plot_height=400, x_range=[""], - tools=tools, sizing_mode="scale_width" - ) - plot.fill_boxplot( - self.plots["sigma_inspect"], - self.sources["sigma_source"], - prefix="sigma", - tooltips=sigma_boxplot_tooltips, - tooltips_formatters=sigma_boxplot_tooltips_formatters) - self.doc.add_root(self.plots["sigma_inspect"]) - - # s plots - self.plots["s10_inspect"] = figure( - name="s10_inspect", - title="", - plot_width=900, plot_height=400, x_range=[""], - tools=tools, sizing_mode='scale_width' - ) - plot.fill_boxplot( - self.plots["s10_inspect"], - self.sources["s10_source"], - prefix="s10", - tooltips=s10_boxplot_tooltips, - tooltips_formatters=s10_boxplot_tooltips_formatters) - s10_tab_inspect = Panel( - child=self.plots["s10_inspect"], - title="Base 10") - - self.plots["s2_inspect"] = figure( - name="s2_inspect", - title="", - plot_width=900, plot_height=400, x_range=[""], - tools=tools, sizing_mode='scale_width' - ) - plot.fill_boxplot( - self.plots["s2_inspect"], self.sources["s2_source"], prefix="s2", - tooltips=s2_boxplot_tooltips, - tooltips_formatters=s2_boxplot_tooltips_formatters - ) - s2_tab_inspect = Panel(child=self.plots["s2_inspect"], title="Base 2") - - s_tabs_inspect = Tabs( - name="s_tabs_inspect", - tabs=[s10_tab_inspect, s2_tab_inspect], tabs_location="below" - ) - self.doc.add_root(s_tabs_inspect) - - def setup_widgets(self): - - # Generation of selectable items - - # Dict contains all inspectable runs (maps display strings to timestamps) - # The dict structure allows to get the timestamp from the display string - # in O(1) - self.runs_dict = self.gen_runs_selection() - - # Dict maps display strings to column names for the different factors - # (var, backend, test) - self.factors_dict = { - "Variables": "variable", - "Backends": "vfc_backend", - "Tests": "test" - } - - # Run selection - - # Contains all options strings - runs_display = list(self.runs_dict.keys()) - # Will be used when updating plots (contains actual number) - self.current_run = self.runs_dict[runs_display[-1]] - # Contains the selected option string, used to update current_n_runs - current_run_display = runs_display[-1] - # This contains only entries matching the run - self.run_data = self.data[self.data["timestamp"] == self.current_run] - - change_run_callback_js = "updateRunMetadata(cb_obj.value);" - - self.widgets["select_run"] = Select( - name="select_run", title="Run :", - value=current_run_display, options=runs_display - ) - self.doc.add_root(self.widgets["select_run"]) - self.widgets["select_run"].on_change("value", self.update_run) - self.widgets["select_run"].js_on_change("value", CustomJS( - code=change_run_callback_js, - args=(dict( - metadata=helper.metadata_to_dict( - helper.get_metadata(self.metadata, self.current_run) - ) - )) - )) - - # Factors selection - - # "Group by" radio - self.widgets["groupby_radio"] = RadioButtonGroup( - name="groupby_radio", - labels=list(self.factors_dict.keys()), active=0 - ) - self.doc.add_root(self.widgets["groupby_radio"]) - # The functions are defined inside the template to avoid writing too - # much JS server side - self.widgets["groupby_radio"].on_change( - "active", - self.update_groupby - ) - - # "Filter by" radio - # Get all possible factors, and remove the one selected in "Group by" - filterby_list = list(self.factors_dict.keys()) - del filterby_list[self.widgets["groupby_radio"].active] - - self.widgets["filterby_radio"] = RadioButtonGroup( - name="filterby_radio", - labels=filterby_list, active=0 - ) - self.doc.add_root(self.widgets["filterby_radio"]) - # The functions are defined inside the template to avoid writing too - # much JS server side - self.widgets["filterby_radio"].on_change( - "active", - self.update_filterby - ) - - # Filter selector - - filterby = self.widgets["filterby_radio"].labels[ - self.widgets["filterby_radio"].active - ] - filterby = self.factors_dict[filterby] - - options = self.run_data.index\ - .get_level_values(filterby).drop_duplicates().tolist() - - self.widgets["select_filter"] = Select( - # We need a different name to avoid collision in the template with - # the runs comparison's widget - name="select_filter", title="Select a filter :", - value=options[0], options=options - ) - self.doc.add_root(self.widgets["select_filter"]) - self.widgets["select_filter"]\ - .on_change("value", self.update_filter) - - # Toggle for outliers filtering - - self.widgets["outliers_filtering_inspect"] = CheckboxGroup( - name="outliers_filtering_inspect", - labels=["Filter outliers"], active=[] - ) - self.doc.add_root(self.widgets["outliers_filtering_inspect"]) - self.widgets["outliers_filtering_inspect"]\ - .on_change("active", self.update_outliers_filtering) - - # Communication methods - # (to send/receive messages to/from master) - - # When received, switch to the run_name in parameter - - def switch_view(self, run_name): - self.widgets["select_run"].value = run_name - - # Constructor - - def __init__(self, master, doc, data, metadata): - - self.master = master - - self.doc = doc - self.data = data - self.metadata = metadata - - self.sources = { - "mu_source": ColumnDataSource(data={}), - "sigma_source": ColumnDataSource(data={}), - "s10_source": ColumnDataSource(data={}), - "s2_source": ColumnDataSource(data={}) - } - - self.plots = {} - self.widgets = {} - - # Setup Bokeh objects - self.setup_plots() - self.setup_widgets() - - # Pass the initial metadata to the template (will be updated in CustomJS - # callbacks). This is required because metadata is not displayed in a - # Bokeh widget, so we can't update this with a server callback. - initial_run = helper.get_metadata(self.metadata, self.current_run) - self.doc.template_variables["initial_timestamp"] = self.current_run - - # At this point, everything should have been initialized, so we can - # show the plots for the first time - self.update_plots() diff --git a/ci/vfc_ci_report/main.py b/ci/vfc_ci_report/main.py deleted file mode 100644 index 4af73d2..0000000 --- a/ci/vfc_ci_report/main.py +++ /dev/null @@ -1,208 +0,0 @@ -# Look for and read all the run files in the current directory (ending with -# .vfcrunh5), and lanch a Bokeh server for the visualization of this data. - -import os -import sys -import time - -import pandas as pd - -from bokeh.plotting import curdoc - -# Local imports from vfc_ci_server -import compare_runs -import inspect_runs -import helper - -########################################################################## - -# Read vfcrun files, and aggregate them in one dataset - -run_files = [f for f in os.listdir(".") if f.endswith(".vfcrun.h5")] - -if len(run_files) == 0: - print( - "Warning [vfc_ci]: Could not find any vfcrun files in the directory. " - "This will result in server errors and prevent you from viewing the report.") - -# These are arrays of Pandas dataframes for now -metadata = [] -data = [] - -for f in run_files: - metadata.append(pd.read_hdf(f, "metadata")) - data.append(pd.read_hdf(f, "data")) - -metadata = pd.concat(metadata).sort_index() -data = pd.concat(data).sort_index() - - -# Generate the display strings for runs (runs ticks) -# By doing this in master, we ensure the homogeneity of display strings -# across all plots -metadata["name"] = metadata.index.to_series().map( - lambda x: helper.get_run_name( - x, - helper.get_metadata(metadata, x)["hash"] - ) -) -helper.reset_run_strings() - -metadata["date"] = metadata.index.to_series().map( - lambda x: time.ctime(x) -) - - -########################################################################## - - -curdoc().title = "Verificarlo Report" - -# Read server arguments -# (this is quite easy because Bokeh server is called through a wrapper, so -# we know exactly what the arguments might be) - -git_repo_linked = False -commit_link = "" - -has_logo = False -logo_url = "" - -for i in range(1, len(sys.argv)): - - # Look for the Git repository remote address - # (if a Git repo is specified, the webpage will contain hyperlinks to the - # repository and the different commits) - if sys.argv[i] == "git": - from urllib.parse import urlparse - - method = sys.argv[i + 1] - address = sys.argv[i + 2] - url = "" - - # Here, address is either the remote URL or the path to the local Git - # repo (depending on the method) - - if method == "url": - # We should directly have a Git URL - url = address - - elif method == "directory": - # Get the remote URL from the local repo - from git import Repo - repo = Repo(address) - url = repo.remotes.origin.url - - else: - raise ValueError( - "Error [vfc_ci]: The specified method to get the Git " - "repository is invalid. Are you calling Bokeh directly " - "instead of using the Verificarlo wrapper ?" - ) - - # At this point, "url" should be set correctly, we can get the repo's - # URL and name, after making sure we're on a Git URL - - parsed_url = urlparse(url) - - path = parsed_url.path.split("/") - if len(path) < 3: - raise ValueError( - "Error [vfc_ci]: The found URL doesn't seem to be pointing " - "to a Git repository (path is too short)" - ) - - repo_name = path[2] - - curdoc().template_variables["repo_url"] = url - curdoc().template_variables["repo_name"] = repo_name - - # We should have a "github.com" or a "*gitlab*" URL - - if parsed_url.netloc == "github.com": - commit_link = "https://%s%s/commit/" \ - % (parsed_url.netloc, parsed_url.path) - - curdoc().template_variables["commit_link"] = commit_link - curdoc().template_variables["git_host"] = "GitHub" - - # Used in Bokeh tooltips - commit_link = commit_link + "@hash" - - # We assume we have a GitLab URL - else: - commit_link = "https://%s%s/-/commit/" \ - % (parsed_url.netloc, parsed_url.path) - - curdoc().template_variables["commit_link"] = commit_link - curdoc().template_variables["git_host"] = "GitLab" - - # Used in Bokeh tooltips - commit_link = commit_link + "@hash" - - git_repo_linked = True - - # Look for a logo URL - # If a logo URL is specified, it will be included in the report's header - if sys.argv[i] == "logo": - curdoc().template_variables["logo_url"] = sys.argv[i + 1] - has_logo = True - - -# After the loop, we know if a repo has been linked, if we have a logo, ... -curdoc().template_variables["git_repo_linked"] = git_repo_linked -curdoc().template_variables["has_logo"] = has_logo - - -########################################################################## - -# Setup report views - -# Define a ViewsMaster class to allow two-ways communication between views. -# This approach by classes allows us to have separate scopes for each view and -# will be useful if we want to add new views at some point in the future -# (instead of having n views with n-1 references each). - -class ViewsMaster: - - # Communication functions - - def go_to_inspect(self, run_name): - self.inspect.switch_view(run_name) - - # Constructor - - def __init__(self, data, metadata, git_repo_linked, commit_link): - - self.data = data - self.metadata = metadata - self.git_repo_linked = git_repo_linked - self.commit_link = commit_link - - # Pass metadata to the template as a JSON string - curdoc().template_variables["metadata"] = self.metadata.to_json( - orient="index") - - # Runs comparison - self.compare = compare_runs.CompareRuns( - master=self, - doc=curdoc(), - data=data, - metadata=metadata, - ) - - # Runs inspection - self.inspect = inspect_runs.InspectRuns( - master=self, - doc=curdoc(), - data=data, - metadata=metadata, - ) - - -views_master = ViewsMaster( - data=data, - metadata=metadata, - git_repo_linked=git_repo_linked, - commit_link=commit_link -) diff --git a/ci/vfc_ci_report/plot.py b/ci/vfc_ci_report/plot.py deleted file mode 100644 index f4bb20f..0000000 --- a/ci/vfc_ci_report/plot.py +++ /dev/null @@ -1,142 +0,0 @@ -# General functions for filling plots with data in all report's views - -from bokeh.plotting import figure -from bokeh.models import HoverTool, TapTool, CustomJS - -from math import pi - - -def fill_dotplot( - plot, source, data_field, - tooltips=None, tooltips_formatters=None, - js_tap_callback=None, server_tap_callback=None, - lines=False, - lower_bound=False -): - - # (Optional) Tooltip and tooltip formatters - if tooltips is not None: - hover = HoverTool(tooltips=tooltips, mode="vline", names=["circle"]) - - if tooltips_formatters is not None: - hover.formatters = tooltips_formatters - - plot.add_tools(hover) - - # (Optional) Add TapTool (for JS tap callback) - if js_tap_callback is not None: - tap = TapTool(callback=CustomJS(code=js_tap_callback)) - plot.add_tools(tap) - - # (Optional) Add segment to represent a lower bound - if lower_bound: - lower_segment = plot.segment( - x0="%s_x" % data_field, y0=data_field, - x1="%s_x" % data_field, y1="%s_lower_bound" % data_field, - source=source, line_color="black" - ) - - # Draw dots (actually Bokeh circles) - circle = plot.circle( - name="circle", - x="%s_x" % data_field, y=data_field, source=source, size=12 - ) - - # (Optional) Draw lines between dots - if lines: - line = plot.line(x="%s_x" % data_field, y=data_field, source=source) - - # (Optional) Add server tap callback - if server_tap_callback is not None: - circle.data_source.selected.on_change("indices", server_tap_callback) - - # Plot appearance - plot.xgrid.grid_line_color = None - plot.ygrid.grid_line_color = None - - plot.yaxis[0].formatter.power_limit_high = 0 - plot.yaxis[0].formatter.power_limit_low = 0 - plot.yaxis[0].formatter.precision = 3 - - plot.xaxis[0].major_label_orientation = pi / 8 - - -def fill_boxplot( - plot, source, - prefix="", - tooltips=None, tooltips_formatters=None, - js_tap_callback=None, server_tap_callback=None -): - - # (Optional) Tooltip and tooltip formatters - if tooltips is not None: - hover = HoverTool(tooltips=tooltips, mode="vline", names=["full_box"]) - - if tooltips_formatters is not None: - hover.formatters = tooltips_formatters - - plot.add_tools(hover) - - # (Optional) Add TapTool (for JS tap callback) - if js_tap_callback is not None: - tap = TapTool(callback=CustomJS(code=js_tap_callback)) - plot.add_tools(tap) - - # Draw boxes (the prefix argument modifies the fields of ColumnDataSource - # that are used) - - if prefix != "": - prefix = "%s_" % prefix - - # Stems - top_stem = plot.segment( - x0="%sx" % prefix, y0="%smax" % prefix, - x1="%sx" % prefix, y1="%squantile75" % prefix, - source=source, line_color="black" - ) - bottom_stem = plot.segment( - x0="%sx" % prefix, y0="%smin" % prefix, - x1="%sx" % prefix, y1="%squantile25" % prefix, - source=source, line_color="black" - ) - - # Boxes - full_box = plot.vbar( - name="full_box", - x="%sx" % prefix, width=0.5, - top="%squantile75" % prefix, bottom="%squantile25" % prefix, - source=source, line_color="black" - ) - bottom_box = plot.vbar( - x="%sx" % prefix, width=0.5, - top="%squantile50" % prefix, bottom="%squantile25" % prefix, - source=source, line_color="black" - ) - - # Mu dot - mu_dot = plot.dot( - x="%sx" % prefix, y="%smu" % prefix, size=30, source=source, - color="black" - ) - - # (Optional) Add server tap callback - if server_tap_callback is not None: - top_stem.data_source.selected.on_change("indices", server_tap_callback) - bottom_stem.data_source.selected.on_change( - "indices", server_tap_callback) - - full_box.data_source.selected.on_change("indices", server_tap_callback) - bottom_box.data_source.selected.on_change( - "indices", server_tap_callback) - - mu_dot.data_source.selected.on_change("indices", server_tap_callback) - - # Plot appearance - plot.xgrid.grid_line_color = None - plot.ygrid.grid_line_color = None - - plot.yaxis[0].formatter.power_limit_high = 0 - plot.yaxis[0].formatter.power_limit_low = 0 - plot.yaxis[0].formatter.precision = 3 - - plot.xaxis[0].major_label_orientation = pi / 8 diff --git a/ci/vfc_ci_report/templates/index.html b/ci/vfc_ci_report/templates/index.html deleted file mode 100644 index 8fea8ec..0000000 --- a/ci/vfc_ci_report/templates/index.html +++ /dev/null @@ -1,482 +0,0 @@ - - - - - Verificarlo Report - - - - - - - - - - {% extends base %} - - - - - {% block contents %} - - - - - - - -
- - {% if has_logo %} - - {% else %} - - {% endif %} -
- - - - - - - {% endblock %} - - - diff --git a/ci/workflow_templates/ci_README.j2.md b/ci/workflow_templates/ci_README.j2.md deleted file mode 100644 index a41ca96..0000000 --- a/ci/workflow_templates/ci_README.j2.md +++ /dev/null @@ -1,7 +0,0 @@ -## Verificarlo CI : `{{dev_branch}}` - -You are on the `{{ci_branch}}` branch, which is automatically updated with the -[Verificarlo](https://github.com/verificarlo/verificarlo) test results from -`{{dev_branch}}` (in the `vfcruns` directory). - -You can start a Verificarlo CI server at anytime using the run files of this branch. diff --git a/ci/workflow_templates/gitlab-ci.j2.yml b/ci/workflow_templates/gitlab-ci.j2.yml deleted file mode 100644 index de26aeb..0000000 --- a/ci/workflow_templates/gitlab-ci.j2.yml +++ /dev/null @@ -1,42 +0,0 @@ -# This workflow will be executed when {{dev_branch}} is updated: -# it will run the configured tests and upload the results on vfc_ci_master. - -image: verificarlo/verificarlo - - -stages: - - run_verificarlo_tests - - -run_verificarlo_tests: - stage: run_verificarlo_tests - - before_script: - - git remote set-url origin https://{{username}}:${CI_PUSH_TOKEN}@{{remote_url}}.git - - git config --global user.email "{{email}}" - - git config --global user.name "{{username}}" - - script: - # We will probably drop these installations when integrating CI into - # Verificarlo - - pip install numpy scipy pandas bokeh jinja2 tables GitPython - - apt update - - apt install wget - - wget https://raw.githubusercontent.com/verificarlo/significantdigits/main/sigdigits.py -P /usr/local/lib/python3.8/dist-packages - - - ./vfc_ci test -g -r - - git_hash=$(git rev-parse --short "$CI_COMMIT_SHA") - - git fetch --all - - git checkout -b {{ci_branch}} origin/{{ci_branch}} - - mkdir -p vfcruns - - mv *.vfcrun.h5 vfcruns - - git add vfcruns/* - - git commit -m "[auto] New test results for commit ${git_hash}" - - git push - - rules: - - if: '$CI_COMMIT_BRANCH == "{{dev_branch}}"' - - artifacts: - paths: - - "*.vfcraw.h5" diff --git a/ci/workflow_templates/vfc_test_workflow.j2.yml b/ci/workflow_templates/vfc_test_workflow.j2.yml deleted file mode 100644 index b39e21e..0000000 --- a/ci/workflow_templates/vfc_test_workflow.j2.yml +++ /dev/null @@ -1,57 +0,0 @@ -# This workflow will be executed when {{dev_branch}} is updated: -# it will run the configured tests and upload the results on {{ci_branch}}. - -name: "Verificarlo CI ({{dev_branch}})" - -on: - # Triggers the workflow when {{dev_branch}} is updated - push: - branches: [ {{dev_branch}} ] - - workflow_dispatch: - - -jobs: - run_verificarlo_tests: - runs-on: ubuntu-latest - container: verificarlo/verificarlo - - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - # We will probably drop these installations when integrating CI into - # Verificarlo - - name: Install Python requirements - run: | - pip install numpy scipy pandas bokeh jinja2 tables GitPython - apt update - apt install wget - wget https://raw.githubusercontent.com/verificarlo/significantdigits/main/sigdigits.py -P /usr/local/lib/python3.8/dist-packages - - - name: Run tests - # We assume the script is included in the repo for now - # (we'll probably want to remove "./" if the script ends up being integrated - # in Verificarlo and becomes available system-wide) - run: ./vfc_ci test -g -r - - - name: Commit test results - run: | - git_hash=$(git rev-parse --short "$GITHUB_SHA") - - git config --local user.email "action@github.com" - git config --local user.name "GitHub Action" - - git checkout {{ci_branch}} - mkdir -p vfcruns - mv *.vfcrun.h5 vfcruns - git add vfcruns/* - git commit -m "[auto] New test results for commit ${git_hash}" - git push - - - name: Upload raw results as artifacts - uses: actions/upload-artifact@v2 - with: - {% raw %}name: ${{github.sha}}.vfcraw{% endraw %} - path: ./*.vfcraw.h5 diff --git a/tests/vfc_test_h5.cpp b/tests/vfc_test_h5.cpp index d7f3c8b..e567b9f 100644 --- a/tests/vfc_test_h5.cpp +++ b/tests/vfc_test_h5.cpp @@ -150,26 +150,33 @@ int test_cycle(H5File file, int cycle, std::string version, } int main(int argc, char **argv) { - if (argc != 3) { + if (argc != 2) { std::cerr << "Execute from within '/'" << std::endl; - std::cerr << "usage: test_h5 " << std::endl; + std::cerr << "usage: vfc_test_h5 " << std::endl; return 1; } - std::string version(argv[1]); - std::vector cycles_list = get_cycles_list(argv[2]); + std::vector cycles_list = get_cycles_list(argv[1]); H5File file(FILE_NAME, H5F_ACC_RDONLY); vfc_probes probes = vfc_init_probes(); probes = vfc_init_probes(); + std::vector versions = {"maponia3", "sm1", "sm2", "sm3", "sm4"}; + bool ok; for (int i = 0; i < cycles_list.size(); i++) { - ok = test_cycle(file, cycles_list[i], version, &probes); - if (ok) { - std::cout << "ok -- cycle " << std::to_string(i) << std::endl; - } else { - std::cerr << "failed -- cycle " << std::to_string(i) << std::endl; + + for(const auto& version: versions) { + + ok = test_cycle(file, cycles_list[i], version, &probes); + if (ok) { + std::cout << "ok -- cycle " << std::to_string(i) << std::endl; + } else { + std::cerr << "failed -- cycle " << std::to_string(i) << std::endl; + } + } + } vfc_dump_probes(&probes); diff --git a/vfc_ci b/vfc_ci deleted file mode 100755 index be0afdb..0000000 --- a/vfc_ci +++ /dev/null @@ -1,199 +0,0 @@ -#!/usr/bin/env python3 - -# This is the entry point of the Verificarlo CI command line interface, which is -# based on argparse and this article : -# https://mike.depalatis.net/blog/simplifying-argparse.html -# From here, 3 subcommands can be called : -# - setup : create a vfc_ci branch and workflow on the current Git repo -# - test : run and export test results according to the vfc_tests_config.json -# - serve : launch a Bokeh server to visualize run results - -import argparse - -########################################################################## - -# Parameters validation helpers - - -def is_port(string): - value = int(string) - if value < 0 or value > 65535: - raise argparse.ArgumentTypeError("Value has to be between 0 and 65535") - return value - - -def is_directory(string): - import os - - isdir = os.path.isdir(string) - if not isdir: - raise argparse.ArgumentTypeError("Directory does not exist") - - return string - - -########################################################################## - - # Subcommand decorator - -cli = argparse.ArgumentParser( - description="Define, run, automatize, and visualize custom Verificarlo tests." -) -subparsers = cli.add_subparsers(dest="subcommand") - - -def subcommand(description="", args=[], parent=subparsers): - def decorator(func): - parser = parent.add_parser(func.__name__, description=description) - for arg in args: - parser.add_argument(*arg[0], **arg[1]) - parser.set_defaults(func=func) - return decorator - - -def argument(*name_or_flags, **kwargs): - return ([*name_or_flags], kwargs) - - -########################################################################## - - # "setup" subcommand - -@subcommand( - description="Create an automated workflow to execute Verificarlo tests.", - args=[ - argument( - "git_host", - help=""" - specify where your repository is hosted - """, - choices=["github", "gitlab"] - ) - ] -) -def setup(args): - import ci.setup - ci.setup.setup(args.git_host) - - # "test" subcommand - - -@subcommand( - description="Execute predefined Verificarlo tests and save their results.", - args=[ - argument( - "-g", "--is-git-commit", - help=""" - When specified, the last Git commit of the local repository (working - directory) will be fetched and associated with the run. - """, - action="store_true" - ), - argument( - "-r", "--export-raw-results", - help=""" - Specify if an additional HDF5 file containing the raw results must be - exported. - """, - action="store_true" - ), - argument( - "-d", "--dry-run", - help=""" - Perform a dry run by not saving the test results. - """, - action="store_true" - ) - ] -) -def test(args): - import ci.test - ci.test.run(args.is_git_commit, args.export_raw_results, args.dry_run) - - # "serve" subcommand - - -@subcommand( - description=""" - Start a server to visualize Verificarlo test results. - """, - args=[ - argument( - "-s", "--show", - help=""" - Specify if the report must be opened in the browser at server - startup. - """, - action="store_true" - ), - argument( - "-gd", "--git-directory", - help=""" - Path to a local Git repository. The report will be linked to the - remote URL (GitHub and GitLab are supported). - """, - type=is_directory - ), - argument( - "-gu", "--git-url", - help=""" - GitHub or GitLab repository URL. The report will be linked to this - URL. - """, - type=str - ), - argument( - "-p", "--port", - help=""" - The port on which the server will run. Defaults to 8080. - """, - type=is_port, - default=8080 - ), - argument( - "-a", "--allow-origin", - help=""" - The origin (URL) from which the report will be accessible. - Port number must not be specified. Defaults to * (allow everything). - """, - type=str, - default="*" - ), - argument( - "-l", "--logo", - help=""" - Specify the URL of an image to be displayed in the report header. - """, - type=str - ) - ] -) -def serve(args): - - # git_directory and git_url are supposed to be exclusive - if args.git_directory is not None and args.git_url is not None: - raise argparse.ArgumentTypeError( - "\"-gd\" / \"--git-directory\" and \"-gu\" / \"--git-url\" are " - "mutually exclusive. Please make sure to use at most one of them." - ) - - import ci.serve - ci.serve.serve( - args.show, - args.git_directory, - args.git_url, - args.port, - args.allow_origin, - args.logo - ) - - -############################################################################### - - # Main command group and entry point -if __name__ == "__main__": - args = cli.parse_args() - if args.subcommand is None: - cli.print_help() - else: - args.func(args) diff --git a/vfc_tests_config.json b/vfc_tests_config.json index 5c61a1e..c5119a0 100644 --- a/vfc_tests_config.json +++ b/vfc_tests_config.json @@ -1,69 +1,10 @@ + { "make_command": "make -f Makefile.verificarlo", "executables": [ { "executable": "bin/vfc_test_h5", - "parameters" : "maponia3 vfc_ci_cycles.txt", - "vfc_backends": [ - { - "name": "libinterflop_mca.so", - "repetitions": 50 - }, - { - "name": "libinterflop_mca.so --mode=rr", - "repetitions": 50 - } - ] - }, - - { - "executable": "bin/vfc_test_h5", - "parameters" : "sm1 vfc_ci_cycles.txt", - "vfc_backends": [ - { - "name": "libinterflop_mca.so", - "repetitions": 50 - }, - { - "name": "libinterflop_mca.so --mode=rr", - "repetitions": 50 - } - ] - }, - - { - "executable": "bin/vfc_test_h5", - "parameters" : "sm2 vfc_ci_cycles.txt", - "vfc_backends": [ - { - "name": "libinterflop_mca.so", - "repetitions": 50 - }, - { - "name": "libinterflop_mca.so --mode=rr", - "repetitions": 50 - } - ] - }, - - { - "executable": "bin/vfc_test_h5", - "parameters" : "sm3 vfc_ci_cycles.txt", - "vfc_backends": [ - { - "name": "libinterflop_mca.so", - "repetitions": 50 - }, - { - "name": "libinterflop_mca.so --mode=rr", - "repetitions": 50 - } - ] - }, - - { - "executable": "bin/vfc_test_h5", - "parameters" : "sm4 vfc_ci_cycles.txt", + "parameters" : "vfc_ci_cycles.txt", "vfc_backends": [ { "name": "libinterflop_mca.so",