diff --git a/.github/workflows/vfc_test_workflow.yml b/.github/workflows/vfc_test_workflow.yml
index 61bad51..edc585c 100644
--- a/.github/workflows/vfc_test_workflow.yml
+++ b/.github/workflows/vfc_test_workflow.yml
@@ -21,35 +21,24 @@ jobs:
with:
fetch-depth: 0
- # We will probably drop these installations when integrating CI into
- # Verificarlo
- - name: Install Python requirements
+ - name: Install HDF5 requirements
run: |
- pip install numpy scipy pandas bokeh jinja2 tables GitPython
apt update
apt install -y wget libhdf5-dev g++
- wget https://raw.githubusercontent.com/verificarlo/significantdigits/main/sigdigits.py -P /usr/local/lib/python3.8/dist-packages
-
- name: Run tests
- # We assume the script is included in the repo for now
- # (we'll probably want to remove "./" if the script ends up being integrated
- # in Verificarlo and becomes available system-wide)
- run: ./vfc_ci test -g -r
+ run: vfc_ci test -g -r
- name: Commit test results
run: |
git_hash=$(git rev-parse --short "$GITHUB_SHA")
-
git config --local user.email "action@github.com"
git config --local user.name "GitHub Action"
-
git checkout vfc_ci_dev
mkdir -p vfcruns
mv *.vfcrun.h5 vfcruns
git add vfcruns/*
git commit -m "[auto] New test results for commit ${git_hash}"
git push
-
- name: Upload raw results as artifacts
uses: actions/upload-artifact@v2
with:
diff --git a/Makefile.verificarlo b/Makefile.verificarlo
index b4b5603..b093482 100644
--- a/Makefile.verificarlo
+++ b/Makefile.verificarlo
@@ -20,7 +20,7 @@ ifeq ($(MKL),-DMKL)
ifeq ($(ENV),INTEL)
LFLAGS = -mkl=sequential # implicit
else
- LFLAGS = $(H5LFLAGS)
+ LFLAGS = $(H5LFLAGS) -lvfc_probes
endif
endif
H5CXXFLAGS = $(CXXFLAGS)
diff --git a/ci/__init__.py b/ci/__init__.py
deleted file mode 100644
index e69de29..0000000
diff --git a/ci/serve.py b/ci/serve.py
deleted file mode 100755
index 37d91f2..0000000
--- a/ci/serve.py
+++ /dev/null
@@ -1,28 +0,0 @@
-# Server for the Verificarlo CI report. This is simply a wrapper to avoid
-# calling Bokeh directly.
-
-import os
-
-
-def serve(show, git_directory, git_url, port, allow_origin, logo_url):
-
- # Prepare arguments
- show = "--show" if show else ""
-
- git = ""
- if git_directory is not None:
- git = "git directory %s" % git_directory
- if git_url is not None:
- git = "git url %s" % git_url
-
- logo = ""
- if logo_url is not None:
- logo = "logo %s" % logo_url
-
- dirname = os.path.dirname(__file__)
-
- # Call the "bokeh serve" command on the system
- command = "bokeh serve %s/vfc_ci_report %s --allow-websocket-origin=%s:%s --port %s --args %s %s" \
- % (dirname, show, allow_origin, port, port, git, logo)
-
- os.system(command)
diff --git a/ci/setup.py b/ci/setup.py
deleted file mode 100644
index 1bd50b3..0000000
--- a/ci/setup.py
+++ /dev/null
@@ -1,146 +0,0 @@
-# Helper script to set up Verificarlo CI on the current branch
-
-import git
-
-import sys
-import os
-from jinja2 import Environment, FileSystemLoader
-
-##########################################################################
-
-# Helper functions
-
-
-def gen_readme(dev_branch, ci_branch):
-
- # Init template loader
- path = os.path.dirname(os.path.abspath(__file__))
- env = Environment(loader=FileSystemLoader(path))
- template = env.get_template("workflow_templates/ci_README.j2.md")
-
- # Render template
- render = template.render(dev_branch=dev_branch, ci_branch=ci_branch)
-
- # Write template
- with open("README.md", "w") as fh:
- fh.write(render)
-
-
-def gen_workflow(git_host, dev_branch, ci_branch, repo):
-
- # Init template loader
- path = os.path.dirname(os.path.abspath(__file__))
- env = Environment(loader=FileSystemLoader(path))
-
- if git_host == "github":
- # Load template
- template = env.get_template(
- "workflow_templates/vfc_test_workflow.j2.yml")
-
- # Render it
- render = template.render(dev_branch=dev_branch, ci_branch=ci_branch)
-
- # Write the file
- filename = ".github/workflows/vfc_test_workflow.yml"
- os.makedirs(os.path.dirname(filename), exist_ok=True)
- with open(filename, "w") as fh:
- fh.write(render)
-
- if git_host == "gitlab":
- template = env.get_template("workflow_templates/gitlab-ci.j2.yml")
-
- # Ask for the user who will run the jobs (Gitlab specific)
- username = input(
- "[vfc_ci] Enter the name of the user who will run the CI jobs:")
- email = input(
- "[vfc_ci] Enter the e-mail of the user who will run the CI jobs:")
-
- remote_url = repo.remotes[0].config_reader.get("url")
- remote_url = remote_url.replace("http://", "")
- remote_url = remote_url.replace("https://", "")
-
- render = template.render(
- dev_branch=dev_branch,
- ci_branch=ci_branch,
- username=username,
- email=email,
- remote_url=remote_url
- )
-
- filename = ".gitlab-ci.yml"
- with open(filename, "w") as fh:
- fh.write(render)
-
-
-##########################################################################
-
-def setup(git_host):
-
- # Init repo and make sure that the workflow setup is possible
-
- repo = git.Repo(".")
- repo.remotes.origin.fetch()
-
- # Make sure that repository is clean
- assert(not repo.is_dirty()), "Error [vfc_ci]: Unstaged changes detected " \
- "in your work tree."
-
- dev_branch = repo.active_branch
- dev_branch_name = str(dev_branch)
- dev_remote = dev_branch.tracking_branch()
-
- # Make sure that the active branch (on which to setup the workflow) has a
- # remote
- assert(dev_remote is not None), "Error [vfc_ci]: The current branch doesn't " \
- "have a remote."
-
- # Make sure that we are not behind the remote (so we can push safely later)
- rev = "%s...%s" % (dev_branch_name, str(dev_remote))
- commits_behind = list(repo.iter_commits(rev))
- assert(commits_behind == []), "Error [vfc_ci]: The local branch seems " \
- "to be at least one commit behind remote."
-
- # Commit the workflow on the current (dev) branch
-
- ci_branch_name = "vfc_ci_%s" % dev_branch_name
- gen_workflow(git_host, dev_branch_name, ci_branch_name, repo)
- repo.git.add(".")
- repo.index.commit("[auto] Set up Verificarlo CI on this branch")
- repo.remote(name="origin").push()
-
- # Create the CI branch (orphan branch with a readme on it)
- # (see : https://github.com/gitpython-developers/GitPython/issues/615)
-
- repo.head.reference = git.Head(repo, "refs/heads/" + ci_branch_name)
-
- repo.index.remove(["*"])
- gen_readme(dev_branch_name, ci_branch_name)
- repo.index.add(["README.md"])
-
- repo.index.commit(
- "[auto] Create the Verificarlo CI branch for %s" % dev_branch_name,
- parent_commits=None
- )
- repo.remote(name="origin").push(
- refspec="%s:%s" % (ci_branch_name, ci_branch_name)
- )
-
- # Force checkout back to the original (dev) branch
- repo.git.checkout(dev_branch_name, force=True)
-
- # Print termination messages
-
- print(
- "Info [vfc_ci]: A Verificarlo CI workflow has been setup on "
- "%s." % dev_branch_name
- )
- print(
- "Info [vfc_ci]: Make sure that you have a \"vfc_tests_config.json\" on "
- "this branch. You can also perform a \"vfc_ci test\" dry run before "
- "pushing other commits.")
-
- if git_host == "gitlab":
- print(
- "Info [vfc_ci]: Since you are using GitLab, make sure that you "
- "have created an access token for the user you specified (registered "
- "as a variable called \"CI_PUSH_TOKEN\" in your repository).")
diff --git a/ci/test.py b/ci/test.py
deleted file mode 100755
index bc38714..0000000
--- a/ci/test.py
+++ /dev/null
@@ -1,367 +0,0 @@
-# This script reads the vfc_tests_config.json file and executes tests accordingly
-# It will also generate a ... .vfcrunh5 file with the results of the run
-
-import sigdigits as sd
-import scipy.stats
-import numpy as np
-import pandas as pd
-import os
-import json
-
-import calendar
-import time
-
-# Forcing an older pickle protocol allows backwards compatibility when reading
-# HDF5 written in 3.8+ using an older version of Python
-import pickle
-pickle.HIGHEST_PROTOCOL = 4
-
-
-# Magic numbers
-min_pvalue = 0.05
-max_zscore = 3
-
-
-##########################################################################
-
-# Helper functions
-
-# Read a CSV file outputted by vfc_probe as a Pandas dataframe
-def read_probes_csv(filepath, backend, warnings, execution_data):
-
- try:
- results = pd.read_csv(filepath)
-
- except FileNotFoundError:
- print(
- "Warning [vfc_ci]: Probes not found, your code might have crashed "
- "or you might have forgotten to call vfc_dump_probes"
- )
- warnings.append(execution_data)
- return pd.DataFrame(
- columns=["test", "variable", "values", "vfc_backend"]
- )
-
- except Exception:
- print(
- "Warning [vfc_ci]: Your probes could not be read for some unknown "
- "reason"
- )
- warnings.append(execution_data)
- return pd.DataFrame(
- columns=["test", "variable", "values", "vfc_backend"]
- )
-
- if len(results) == 0:
- print(
- "Warning [vfc_ci]: Probes empty, it looks like you have dumped "
- "them without calling vfc_put_probe"
- )
- warnings.append(execution_data)
-
- # Once the CSV has been opened and validated, return its content
- results["value"] = results["value"].apply(lambda x: float.fromhex(x))
- results.rename(columns={"value": "values"}, inplace=True)
-
- results["vfc_backend"] = backend
-
- return results
-
-
-# Wrappers to sd.significant_digits (returns results in base 2)
-
-def significant_digits(x):
-
- # If the null hypothesis is rejected, call sigdigits with the General
- # formula:
- if x.pvalue < min_pvalue:
- # In a pandas DF, "values" actually refers to the array of columns, and
- # not the column named "values"
- distribution = x.values[3]
- distribution = distribution.reshape(len(distribution), 1)
-
- # The distribution's empirical average will be used as the reference
- mu = np.array([x.mu])
-
- s = sd.significant_digits(
- distribution,
- mu,
- precision=sd.Precision.Relative,
- method=sd.Method.General,
-
- probability=0.9,
- confidence=0.95
- )
-
- # s is returned inside a list
- return s[0]
-
- # Else, manually compute sMCA (Stott-Parker formula)
- else:
- return -np.log2(np.absolute(x.sigma / x.mu))
-
-
-def significant_digits_lower_bound(x):
- # If the null hypothesis is rejected, no lower bound
- if x.pvalue < min_pvalue:
- return x.s2
-
- # Else, the lower bound will be computed with p= .9 alpha-1=.95
- else:
- distribution = x.values[3]
- distribution = distribution.reshape(len(distribution), 1)
-
- mu = np.array([x.mu])
-
- s = sd.significant_digits(
- distribution,
- mu,
- precision=sd.Precision.Relative,
- method=sd.Method.CNH,
-
- probability=0.9,
- confidence=0.95
- )
-
- return s[0]
-
-
-##########################################################################
-
- # Main functions
-
-
-# Open and read the tests config file
-def read_config():
- try:
- with open("vfc_tests_config.json", "r") as file:
- data = file.read()
-
- except FileNotFoundError as e:
- e.strerror = "Error [vfc_ci]: This file is required to describe the tests "\
- "to run and generate a Verificarlo run file"
- raise e
-
- return json.loads(data)
-
-
-# Set up metadata
-def generate_metadata(is_git_commit):
-
- # Metadata and filename are initiated as if no commit was associated
- metadata = {
- "timestamp": calendar.timegm(time.gmtime()),
- "is_git_commit": is_git_commit,
- "hash": "",
- "author": "",
- "message": ""
- }
-
- if is_git_commit:
- print("Fetching metadata from last commit...")
- from git import Repo
-
- repo = Repo(".")
- head_commit = repo.head.commit
-
- metadata["timestamp"] = head_commit.authored_date
-
- metadata["hash"] = str(head_commit)[0:7]
- metadata["author"] = "%s <%s>" \
- % (str(head_commit.author), head_commit.author.email)
- metadata["message"] = head_commit.message.split("\n")[0]
-
- return metadata
-
-
-# Execute tests and collect results in a Pandas dataframe (+ dataprocessing)
-def run_tests(config):
-
- # Run the build command
- print("Info [vfc_ci]: Building tests...")
- os.system(config["make_command"])
-
- # This is an array of Pandas dataframes for now
- data = []
-
- # Create tmp folder to export results
- os.system("mkdir .vfcruns.tmp")
- n_files = 0
-
- # This will contain all executables/repetition numbers from which we could
- # not get any data
- warnings = []
-
- # Tests execution loop
- for executable in config["executables"]:
- print("Info [vfc_ci]: Running executable :",
- executable["executable"], "...")
-
- parameters = ""
- if "parameters" in executable:
- parameters = executable["parameters"]
-
- for backend in executable["vfc_backends"]:
-
- export_backend = "VFC_BACKENDS=\"" + backend["name"] + "\" "
- command = "./" + executable["executable"] + " " + parameters
-
- repetitions = 1
- if "repetitions" in backend:
- repetitions = backend["repetitions"]
-
- # Run test repetitions and save results
- for i in range(repetitions):
- file = ".vfcruns.tmp/%s.csv" % str(n_files)
- export_output = "VFC_PROBES_OUTPUT=\"%s\" " % file
- os.system(export_output + export_backend + command)
-
- # This will only be used if we need to append this exec to the
- # warnings list
- execution_data = {
- "executable": executable["executable"],
- "backend": backend["name"],
- "repetition": i + 1
- }
-
- data.append(read_probes_csv(
- file,
- backend["name"],
- warnings,
- execution_data
- ))
-
- n_files = n_files + 1
-
- # Clean CSV output files (by deleting the tmp folder)
- os.system("rm -rf .vfcruns.tmp")
-
- # Combine all separate executions in one dataframe
- data = pd.concat(data, sort=False, ignore_index=True)
- data = data.groupby(["test", "vfc_backend", "variable"])\
- .values.apply(list).reset_index()
-
- # Make sure we have some data to work on
- assert(len(data) != 0), "Error [vfc_ci]: No data have been generated " \
- "by your tests executions, aborting run without writing results file"
-
- return data, warnings
-
- # Data processing
-
-
-def data_processing(data):
-
- data["values"] = data["values"].apply(lambda x: np.array(x).astype(float))
-
- # Get empirical average, standard deviation and p-value
- data["mu"] = data["values"].apply(np.average)
- data["sigma"] = data["values"].apply(np.std)
- data["pvalue"] = data["values"].apply(
- lambda x: scipy.stats.shapiro(x).pvalue)
-
- # Significant digits
- data["s2"] = data.apply(significant_digits, axis=1)
- data["s10"] = data["s2"].apply(lambda x: sd.change_base(x, 10))
-
- # Lower bound of the confidence interval using the sigdigits module
- data["s2_lower_bound"] = data.apply(significant_digits_lower_bound, axis=1)
- data["s10_lower_bound"] = data["s2_lower_bound"].apply(
- lambda x: sd.change_base(x, 10))
-
- # Compute moments of the distribution
- # (including a new distribution obtained by filtering outliers)
- data["values"] = data["values"].apply(np.sort)
-
- data["mu"] = data["values"].apply(np.average)
- data["min"] = data["values"].apply(np.min)
- data["quantile25"] = data["values"].apply(np.quantile, args=(0.25,))
- data["quantile50"] = data["values"].apply(np.quantile, args=(0.50,))
- data["quantile75"] = data["values"].apply(np.quantile, args=(0.75,))
- data["max"] = data["values"].apply(np.max)
-
- data["nsamples"] = data["values"].apply(len)
-
- # Display all executions that resulted in a warning
-
-
-def show_warnings(warnings):
- if len(warnings) > 0:
- print(
- "Warning [vfc_ci]: Some of your runs could not generate any data "
- "(for instance because your code crashed) and resulted in "
- "warnings. Here is the complete list :"
- )
-
- for i in range(0, len(warnings)):
- print("- Warning %s:" % i)
-
- print(" Executable: %s" % warnings[i]["executable"])
- print(" Backend: %s" % warnings[i]["backend"])
- print(" Repetition: %s" % warnings[i]["repetition"])
-
-
-##########################################################################
-
- # Entry point
-
-def run(is_git_commit, export_raw_values, dry_run):
-
- # Get config, metadata and data
- print("Info [vfc_ci]: Reading tests config file...")
- config = read_config()
-
- print("Info [vfc_ci]: Generating run metadata...")
- metadata = generate_metadata(is_git_commit)
-
- data, warnings = run_tests(config)
- show_warnings(warnings)
-
- # Data processing
- print("Info [vfc_ci]: Processing data...")
- data_processing(data)
-
- # Prepare data for export (by creating a proper index and linking run
- # timestamp)
- data = data.set_index(["test", "variable", "vfc_backend"]).sort_index()
- data["timestamp"] = metadata["timestamp"]
-
- filename = metadata["hash"] if is_git_commit else str(
- metadata["timestamp"])
-
- # Prepare metadata for export
- metadata = pd.DataFrame.from_dict([metadata])
- metadata = metadata.set_index("timestamp")
-
- # NOTE : Exporting to HDF5 requires to install "tables" on the system
-
- # Export raw data if needed
- if export_raw_values and not dry_run:
- data.to_hdf(filename + ".vfcraw.h5", key="data")
- metadata.to_hdf(filename + ".vfcraw.h5", key="metadata")
-
- # Export data
- del data["values"]
- if not dry_run:
- data.to_hdf(filename + ".vfcrun.h5", key="data")
- metadata.to_hdf(filename + ".vfcrun.h5", key="metadata")
-
- # Print termination messages
- print(
- "Info [vfc_ci]: The results have been successfully written to "
- "%s.vfcrun.h5."
- % filename
- )
-
- if export_raw_values:
- print(
- "Info [vfc_ci]: A file containing the raw values has also been "
- "created : %s.vfcraw.h5."
- % filename
- )
-
- if dry_run:
- print(
- "Info [vfc_ci]: The dry run flag was enabled, so no files were "
- "actually created."
- )
diff --git a/ci/vfc_ci_report/compare_runs.py b/ci/vfc_ci_report/compare_runs.py
deleted file mode 100644
index d05b676..0000000
--- a/ci/vfc_ci_report/compare_runs.py
+++ /dev/null
@@ -1,542 +0,0 @@
-# Manage the view comparing a variable over different runs
-
-import time
-
-import pandas as pd
-
-from math import pi
-
-from bokeh.plotting import figure, curdoc
-from bokeh.embed import components
-from bokeh.models import Select, ColumnDataSource, Panel, Tabs, HoverTool, \
- TextInput, CheckboxGroup, TapTool, CustomJS
-
-import helper
-import plot
-
-
-##########################################################################
-
-
-class CompareRuns:
-
- # Helper functions related to CompareRuns
-
- # From an array of timestamps, returns the array of runs names (for the x
- # axis ticks), as well as the metadata (in a dict of arrays) associated to
- # this array (for the tooltips)
- def gen_x_series(self, timestamps):
-
- # Initialize the objects to return
- x_series = []
- x_metadata = dict(
- date=[],
- is_git_commit=[],
- hash=[],
- author=[],
- message=[]
- )
-
- # n == 0 means we want all runs, we also make sure not to go out of
- # bound if asked for more runs than we have
- n = self.current_n_runs
- if n == 0 or n > len(timestamps):
- n = len(timestamps)
-
- for i in range(0, n):
- # Get metadata associated to this run
- row_metadata = helper.get_metadata(
- self.metadata, timestamps[-i - 1])
- date = time.ctime(timestamps[-i - 1])
-
- # Fill the x series
- str = row_metadata["name"]
- x_series.insert(0, helper.get_metadata(
- self.metadata, timestamps[-i - 1])["name"])
-
- # Fill the metadata lists
- x_metadata["date"].insert(0, date)
- x_metadata["is_git_commit"].insert(
- 0, row_metadata["is_git_commit"])
- x_metadata["hash"].insert(0, row_metadata["hash"])
- x_metadata["author"].insert(0, row_metadata["author"])
- x_metadata["message"].insert(0, row_metadata["message"])
-
- return x_series, x_metadata
-
- # Plots update function
-
- def update_plots(self):
-
- # Select all data matching current test/var/backend
-
- runs = self.data.loc[[self.widgets["select_test"].value],
- self.widgets["select_var"].value,
- self.widgets["select_backend"].value]
-
- timestamps = runs["timestamp"]
- x_series, x_metadata = self.gen_x_series(timestamps.sort_values())
-
- # Update source
-
- main_dict = runs.to_dict("series")
- main_dict["x"] = x_series
-
- # Add metadata (for tooltip)
- main_dict.update(x_metadata)
-
- # Select the last n runs only
- n = self.current_n_runs
- main_dict = {key: value[-n:] for key, value in main_dict.items()}
-
- # Generate ColumnDataSources for the 3 dotplots
- for stat in ["sigma", "s10", "s2"]:
- dict = {
- "%s_x" % stat: main_dict["x"],
-
- "is_git_commit": main_dict["is_git_commit"],
- "date": main_dict["date"],
- "hash": main_dict["hash"],
- "author": main_dict["author"],
- "message": main_dict["message"],
-
- stat: main_dict[stat],
-
- "nsamples": main_dict["nsamples"],
- }
-
- if stat == "s10" or stat == "s2":
- dict["%s_lower_bound" %
- stat] = main_dict["%s_lower_bound" %
- stat]
-
- # Filter outliers if the box is checked
- if len(self.widgets["outliers_filtering_compare"].active) > 0:
- outliers = helper.detect_outliers(dict[stat])
- dict[stat] = helper.remove_outliers(dict[stat], outliers)
- dict["%s_x" % stat] = helper.remove_outliers(
- dict["%s_x" % stat], outliers)
-
- # Assign ColumnDataSource
- self.sources["%s_source" % stat].data = dict
-
- # Generate ColumnDataSource for the boxplot
- dict = {
- "is_git_commit": main_dict["is_git_commit"],
- "date": main_dict["date"],
- "hash": main_dict["hash"],
- "author": main_dict["author"],
- "message": main_dict["message"],
-
- "x": main_dict["x"],
- "min": main_dict["min"],
- "quantile25": main_dict["quantile25"],
- "quantile50": main_dict["quantile50"],
- "quantile75": main_dict["quantile75"],
- "max": main_dict["max"],
- "mu": main_dict["mu"],
- "pvalue": main_dict["pvalue"],
-
- "nsamples": main_dict["nsamples"]
- }
-
- self.sources["boxplot_source"].data = dict
-
- # Update x axis
-
- helper.reset_x_range(
- self.plots["boxplot"],
- self.sources["boxplot_source"].data["x"]
- )
- helper.reset_x_range(
- self.plots["sigma_plot"],
- self.sources["sigma_source"].data["sigma_x"]
- )
- helper.reset_x_range(
- self.plots["s10_plot"],
- self.sources["s10_source"].data["s10_x"]
- )
- helper.reset_x_range(
- self.plots["s2_plot"],
- self.sources["s2_source"].data["s2_x"]
- )
-
- # Widgets' callback functions
-
- def update_test(self, attrname, old, new):
-
- # If the value is updated by the CustomJS, self.widgets["select_var"].value
- # won't be updated, so we have to look for that case and assign it
- # manually
-
- # "new" should be a list when updated by CustomJS
- if isinstance(new, list):
- # If filtering removed all options, we might have an empty list
- # (in this case, we just skip the callback and do nothing)
- if len(new) > 0:
- new = new[0]
- else:
- return
-
- if new != self.widgets["select_test"].value:
- # The callback will be triggered again with the updated value
- self.widgets["select_test"].value = new
- return
-
- # New list of available vars
- self.vars = self.data.loc[new]\
- .index.get_level_values("variable").drop_duplicates().tolist()
- self.widgets["select_var"].options = self.vars
-
- # Reset var selection if old one is not available in new vars
- if self.widgets["select_var"].value not in self.vars:
- self.widgets["select_var"].value = self.vars[0]
- # The update_var callback will be triggered by the assignment
-
- else:
- # Trigger the callback manually (since the plots need to be updated
- # anyway)
- self.update_var("", "", self.widgets["select_var"].value)
-
- def update_var(self, attrname, old, new):
-
- # If the value is updated by the CustomJS, self.widgets["select_var"].value
- # won't be updated, so we have to look for that case and assign it
- # manually
-
- # new should be a list when updated by CustomJS
- if isinstance(new, list):
- new = new[0]
-
- if new != self.widgets["select_var"].value:
- # The callback will be triggered again with the updated value
- self.widgets["select_var"].value = new
- return
-
- # New list of available backends
- self.backends = self.data.loc[self.widgets["select_test"].value, self.widgets["select_var"].value]\
- .index.get_level_values("vfc_backend").drop_duplicates().tolist()
- self.widgets["select_backend"].options = self.backends
-
- # Reset backend selection if old one is not available in new backends
- if self.widgets["select_backend"].value not in self.backends:
- self.widgets["select_backend"].value = self.backends[0]
- # The update_backend callback will be triggered by the assignment
-
- else:
- # Trigger the callback manually (since the plots need to be updated
- # anyway)
- self.update_backend("", "", self.widgets["select_backend"].value)
-
- def update_backend(self, attrname, old, new):
-
- # Simply update plots, since no other data is affected
- self.update_plots()
-
- def update_n_runs(self, attrname, old, new):
- # Simply update runs selection (value and string display)
- self.widgets["select_n_runs"].value = new
- self.current_n_runs = self.n_runs_dict[self.widgets["select_n_runs"].value]
-
- self.update_plots()
-
- def update_outliers_filtering(self, attrname, old, new):
- self.update_plots()
-
- # Bokeh setup functions
-
- def setup_plots(self):
-
- tools = "pan, wheel_zoom, xwheel_zoom, ywheel_zoom, reset, save"
-
- # Custom JS callback that will be used when tapping on a run
- # Only switches the view, a server callback is required to update plots
- # (defined inside template to avoid bloating server w/ too much JS code)
- js_tap_callback = "goToInspectRuns();"
-
- # Box plot
- self.plots["boxplot"] = figure(
- name="boxplot", title="Variable distribution over runs",
- plot_width=900, plot_height=400, x_range=[""],
- tools=tools, sizing_mode="scale_width"
- )
-
- box_tooltips = [
- ("Git commit", "@is_git_commit"),
- ("Date", "@date"),
- ("Hash", "@hash"),
- ("Author", "@author"),
- ("Message", "@message"),
- ("Min", "@min{%0.18e}"),
- ("Max", "@max{%0.18e}"),
- ("1st quartile", "@quantile25{%0.18e}"),
- ("Median", "@quantile50{%0.18e}"),
- ("3rd quartile", "@quantile75{%0.18e}"),
- ("μ", "@mu{%0.18e}"),
- ("p-value", "@pvalue"),
- ("Number of samples", "@nsamples")
- ]
- box_tooltips_formatters = {
- "@min": "printf",
- "@max": "printf",
- "@quantile25": "printf",
- "@quantile50": "printf",
- "@quantile75": "printf",
- "@mu": "printf"
- }
-
- plot.fill_boxplot(
- self.plots["boxplot"], self.sources["boxplot_source"],
- tooltips=box_tooltips,
- tooltips_formatters=box_tooltips_formatters,
- js_tap_callback=js_tap_callback,
- server_tap_callback=self.inspect_run_callback_boxplot,
- )
- self.doc.add_root(self.plots["boxplot"])
-
- # Sigma plot (bar plot)
- self.plots["sigma_plot"] = figure(
- name="sigma_plot", title="Standard deviation σ over runs",
- plot_width=900, plot_height=400, x_range=[""],
- tools=tools, sizing_mode="scale_width"
- )
-
- sigma_tooltips = [
- ("Git commit", "@is_git_commit"),
- ("Date", "@date"),
- ("Hash", "@hash"),
- ("Author", "@author"),
- ("Message", "@message"),
- ("σ", "@sigma"),
- ("Number of samples", "@nsamples")
- ]
-
- plot.fill_dotplot(
- self.plots["sigma_plot"], self.sources["sigma_source"], "sigma",
- tooltips=sigma_tooltips,
- js_tap_callback=js_tap_callback,
- server_tap_callback=self.inspect_run_callback_sigma,
- lines=True
- )
- self.doc.add_root(self.plots["sigma_plot"])
-
- # s plot (bar plot with 2 tabs)
- self.plots["s10_plot"] = figure(
- name="s10_plot", title="Significant digits s over runs",
- plot_width=900, plot_height=400, x_range=[""],
- tools=tools, sizing_mode="scale_width"
- )
-
- s10_tooltips = [
- ("Git commit", "@is_git_commit"),
- ("Date", "@date"),
- ("Hash", "@hash"),
- ("Author", "@author"),
- ("Message", "@message"),
- ("s", "@s10"),
- ("s lower bound", "@s10_lower_bound"),
- ("Number of samples", "@nsamples")
- ]
-
- plot.fill_dotplot(
- self.plots["s10_plot"], self.sources["s10_source"], "s10",
- tooltips=s10_tooltips,
- js_tap_callback=js_tap_callback,
- server_tap_callback=self.inspect_run_callback_s10,
- lines=True,
- lower_bound=True
- )
- s10_tab = Panel(child=self.plots["s10_plot"], title="Base 10")
-
- self.plots["s2_plot"] = figure(
- name="s2_plot", title="Significant digits s over runs",
- plot_width=900, plot_height=400, x_range=[""],
- tools=tools, sizing_mode="scale_width"
- )
-
- s2_tooltips = [
- ("Git commit", "@is_git_commit"),
- ("Date", "@date"),
- ("Hash", "@hash"),
- ("Author", "@author"),
- ("Message", "@message"),
- ("s", "@s2"),
- ("s lower bound", "@s2_lower_bound"),
- ("Number of samples", "@nsamples")
- ]
-
- plot.fill_dotplot(
- self.plots["s2_plot"], self.sources["s2_source"], "s2",
- tooltips=s2_tooltips,
- js_tap_callback=js_tap_callback,
- server_tap_callback=self.inspect_run_callback_s2,
- lines=True,
- lower_bound=True
- )
- s2_tab = Panel(child=self.plots["s2_plot"], title="Base 2")
-
- s_tabs = Tabs(
- name="s_tabs",
- tabs=[s10_tab, s2_tab],
- tabs_location="below"
- )
-
- self.doc.add_root(s_tabs)
-
- def setup_widgets(self):
-
- # Initial selections
-
- # Test/var/backend combination (we select all first elements at init)
- self.tests = self.data\
- .index.get_level_values("test").drop_duplicates().tolist()
-
- self.vars = self.data.loc[self.tests[0]]\
- .index.get_level_values("variable").drop_duplicates().tolist()
-
- self.backends = self.data.loc[self.tests[0], self.vars[0]]\
- .index.get_level_values("vfc_backend").drop_duplicates().tolist()
-
- # Custom JS callback that will be used client side to filter selections
- filter_callback_js = """
- selector.options = options.filter(e => e.includes(cb_obj.value));
- """
-
- # Test selector widget
-
- # Number of runs to display
- # The dict structure allows us to get int value from the display string
- # in O(1)
- self.n_runs_dict = {
- "Last 3 runs": 3,
- "Last 5 runs": 5,
- "Last 10 runs": 10,
- "All runs": 0
- }
-
- # Contains all options strings
- n_runs_display = list(self.n_runs_dict.keys())
-
- # Will be used when updating plots (contains actual number to diplay)
- self.current_n_runs = self.n_runs_dict[n_runs_display[1]]
-
- # Selector widget
- self.widgets["select_test"] = Select(
- name="select_test", title="Test :",
- value=self.tests[0], options=self.tests
- )
- self.doc.add_root(self.widgets["select_test"])
- self.widgets["select_test"].on_change("value", self.update_test)
- self.widgets["select_test"].on_change("options", self.update_test)
-
- # Filter widget
- self.widgets["test_filter"] = TextInput(
- name="test_filter", title="Tests filter:"
- )
- self.widgets["test_filter"].js_on_change(
- "value",
- CustomJS(
- args=dict(
- options=self.tests,
- selector=self.widgets["select_test"]),
- code=filter_callback_js))
- self.doc.add_root(self.widgets["test_filter"])
-
- # Number of runs to display
-
- self.widgets["select_n_runs"] = Select(
- name="select_n_runs", title="Display :",
- value=n_runs_display[1], options=n_runs_display
- )
- self.doc.add_root(self.widgets["select_n_runs"])
- self.widgets["select_n_runs"].on_change("value", self.update_n_runs)
-
- # Variable selector widget
-
- self.widgets["select_var"] = Select(
- name="select_var", title="Variable :",
- value=self.vars[0], options=self.vars
- )
- self.doc.add_root(self.widgets["select_var"])
- self.widgets["select_var"].on_change("value", self.update_var)
- self.widgets["select_var"].on_change("options", self.update_var)
-
- # Backend selector widget
-
- self.widgets["select_backend"] = Select(
- name="select_backend", title="Verificarlo backend :",
- value=self.backends[0], options=self.backends
- )
- self.doc.add_root(self.widgets["select_backend"])
- self.widgets["select_backend"].on_change("value", self.update_backend)
-
- # Outliers filtering checkbox
-
- self.widgets["outliers_filtering_compare"] = CheckboxGroup(
- name="outliers_filtering_compare",
- labels=["Filter outliers"], active=[]
- )
- self.doc.add_root(self.widgets["outliers_filtering_compare"])
- self.widgets["outliers_filtering_compare"]\
- .on_change("active", self.update_outliers_filtering)
-
- # Communication methods
- # (to send/receive messages to/from master)
-
- # Callback to change view of Inspect runs when data is selected
-
- def inspect_run_callback(self, new, source_name, x_name):
-
- # In case we just unselected everything, then do nothing
- if new == []:
- return
-
- index = new[-1]
- run_name = self.sources[source_name].data[x_name][index]
-
- self.master.go_to_inspect(run_name)
-
- # Wrappers for each plot (since new is the index of the clicked element,
- # it is dependent of the plot because we could have filtered some outliers)
- # There doesn't seem to be an easy way to add custom parameters to a
- # Bokeh callback, so using wrappers seems to be the best solution for now
-
- def inspect_run_callback_boxplot(self, attr, old, new):
- self.inspect_run_callback(new, "boxplot_source", "x")
-
- def inspect_run_callback_sigma(self, attr, old, new):
- self.inspect_run_callback(new, "sigma_source", "sigma_x")
-
- def inspect_run_callback_s2(self, attr, old, new):
- self.inspect_run_callback(new, "s2_source", "s2_x")
-
- def inspect_run_callback_s10(self, attr, old, new):
- self.inspect_run_callback(new, "s10_source", "s10_x")
-
- # Constructor
-
- def __init__(self, master, doc, data, metadata):
-
- self.master = master
-
- self.doc = doc
- self.data = data
- self.metadata = metadata
-
- self.sources = {
- "boxplot_source": ColumnDataSource(data={}),
- "sigma_source": ColumnDataSource(data={}),
- "s10_source": ColumnDataSource(data={}),
- "s2_source": ColumnDataSource(data={})
- }
-
- self.plots = {}
- self.widgets = {}
-
- # Setup Bokeh objects
- self.setup_plots()
- self.setup_widgets()
-
- # At this point, everything should have been initialized, so we can
- # show the plots for the first time
- self.update_plots()
diff --git a/ci/vfc_ci_report/helper.py b/ci/vfc_ci_report/helper.py
deleted file mode 100644
index f258959..0000000
--- a/ci/vfc_ci_report/helper.py
+++ /dev/null
@@ -1,170 +0,0 @@
-# General helper functions for both compare_runs and compare_variables
-
-import calendar
-import time
-from itertools import compress
-
-import numpy as np
-
-# Magic numbers
-max_ticks = 15
-max_zscore = 3
-
-##########################################################################
-
-
-# From a timestamp, return the associated metadata as a Pandas serie
-def get_metadata(metadata, timestamp):
- return metadata.loc[timestamp]
-
-
-# Convert a metadata Pandas series to a JS readable dict
-def metadata_to_dict(metadata):
- dict = metadata.to_dict()
-
- # JS doesn't accept True for booleans, and Python doesn't accept true
- # (because of the caps) => using an integer is a portable solution
- dict["is_git_commit"] = 1 if dict["is_git_commit"] else 0
-
- dict["date"] = time.ctime(metadata.name)
-
- return dict
-
-
-# Return a string that indicates the elapsed time since the run, used as the
-# x-axis tick in "Compare runs" or when selecting run in "Inspect run"
-def get_run_name(timestamp, hash):
-
- gmt = time.gmtime()
- now = calendar.timegm(gmt)
- diff = now - timestamp
-
- # Special case : < 1 minute (return string directly)
- if diff < 60:
- str = "Less than a minute ago"
-
- if hash != "":
- str = str + " (%s)" % hash
-
- if str == get_run_name.previous:
- get_run_name.counter = get_run_name.counter + 1
- str = "%s (%s)" % (str, get_run_name.counter)
- else:
- get_run_name.counter = 0
- get_run_name.previous = str
-
- return str
-
- # < 1 hour
- if diff < 3600:
- n = int(diff / 60)
- str = "%s minute%s ago"
- # < 1 day
- elif diff < 86400:
- n = int(diff / 3600)
- str = "%s hour%s ago"
- # < 1 week
- elif diff < 604800:
- n = int(diff / 86400)
- str = "%s day%s ago"
- # < 1 month
- elif diff < 2592000:
- n = int(diff / 604800)
- str = "%s week%s ago"
- # > 1 month
- else:
- n = diff / 2592000
- str = "%s month%s ago"
-
- plural = ""
- if n != 1:
- plural = "s"
-
- str = str % (n, plural)
-
- # We might want to add the git hash
- if hash != "":
- str = str + " (%s)" % hash
-
- # Finally, check for duplicate with previously generated string
- if str == get_run_name.previous:
- # Increment the duplicate counter and add it to str
- get_run_name.counter = get_run_name.counter + 1
- str = "%s (%s)" % (str, get_run_name.counter)
-
- else:
- # No duplicate, reset both previously generated str and duplicate
- # counter
- get_run_name.counter = 0
- get_run_name.previous = str
-
- return str
-
-
-# These external variables will store data about the last generated string to
-# avoid duplicates (assuming the runs are sorted by time)
-get_run_name.counter = 0
-get_run_name.previous = ""
-
-
-def reset_run_strings():
- get_run_name.counter = 0
- get_run_name.previous = ""
-
-
-# Update all the x-ranges from a dict of plots
-def reset_x_range(plot, x_range):
- plot.x_range.factors = x_range
-
- if len(x_range) < max_ticks:
- plot.xaxis.major_tick_line_color = "#000000"
- plot.xaxis.minor_tick_line_color = "#000000"
-
- plot.xaxis.major_label_text_font_size = "8pt"
-
- else:
- plot.xaxis.major_tick_line_color = None
- plot.xaxis.minor_tick_line_color = None
-
- plot.xaxis.major_label_text_font_size = "0pt"
-
-
-# Return an array of booleans that indicate which elements are outliers
-# (True means element is not an outlier and must be kept)
-def detect_outliers(array, max_zscore=max_zscore):
- if len(array) <= 2:
- return [True] * len(array)
-
- median = np.median(array)
- std = np.std(array)
- if std == 0:
- return array
- distance = abs(array - median)
- # Array of booleans with elements to be filtered
- outliers_array = distance < max_zscore * std
-
- return outliers_array
-
-
-def remove_outliers(array, outliers):
- return list(compress(array, outliers))
-
-
-def remove_boxplot_outliers(dict, outliers, prefix):
- outliers = detect_outliers(dict["%s_max" % prefix])
-
- dict["%s_x" % prefix] = remove_outliers(dict["%s_x" % prefix], outliers)
-
- dict["%s_min" % prefix] = remove_outliers(
- dict["%s_min" % prefix], outliers)
- dict["%s_quantile25" % prefix] = remove_outliers(
- dict["%s_quantile25" % prefix], outliers)
- dict["%s_quantile50" % prefix] = remove_outliers(
- dict["%s_quantile50" % prefix], outliers)
- dict["%s_quantile75" % prefix] = remove_outliers(
- dict["%s_quantile75" % prefix], outliers)
- dict["%s_max" % prefix] = remove_outliers(
- dict["%s_max" % prefix], outliers)
- dict["%s_mu" % prefix] = remove_outliers(dict["%s_mu" % prefix], outliers)
-
- dict["nsamples"] = remove_outliers(dict["nsamples"], outliers)
diff --git a/ci/vfc_ci_report/inspect_runs.py b/ci/vfc_ci_report/inspect_runs.py
deleted file mode 100644
index f53ab17..0000000
--- a/ci/vfc_ci_report/inspect_runs.py
+++ /dev/null
@@ -1,570 +0,0 @@
-# Manage the view comparing the variables of a run
-
-from math import pi
-from functools import partial
-
-import pandas as pd
-import numpy as np
-
-from bokeh.plotting import figure, curdoc
-from bokeh.embed import components
-from bokeh.models import Select, ColumnDataSource, Panel, Tabs, HoverTool,\
- RadioButtonGroup, CheckboxGroup, CustomJS
-
-import helper
-import plot
-
-
-##########################################################################
-
-
-class InspectRuns:
-
- # Helper functions related to InspectRun
-
- # Returns a dictionary mapping user-readable strings to all run timestamps
- def gen_runs_selection(self):
-
- runs_dict = {}
-
- # Iterate over timestamp rows (runs) and fill dict
- for row in self.metadata.iloc:
- # The syntax used by pandas makes this part a bit tricky :
- # row.name is the index of metadata (so it refers to the
- # timestamp), whereas row["name"] is the column called "name"
- # (which is the display string used for the run)
-
- # runs_dict[run's name] = run's timestamp
- runs_dict[row["name"]] = row.name
-
- return runs_dict
-
- def gen_boxplot_tooltips(self, prefix):
- return [
- ("Name", "@%s_x" % prefix),
- ("Min", "@" + prefix + "_min{%0.18e}"),
- ("Max", "@" + prefix + "_max{%0.18e}"),
- ("1st quartile", "@" + prefix + "_quantile25{%0.18e}"),
- ("Median", "@" + prefix + "_quantile50{%0.18e}"),
- ("3rd quartile", "@" + prefix + "_quantile75{%0.18e}"),
- ("μ", "@" + prefix + "_mu{%0.18e}"),
- ("Number of samples (tests)", "@nsamples")
- ]
-
- def gen_boxplot_tooltips_formatters(self, prefix):
- return {
- "@%s_min" % prefix: "printf",
- "@%s_max" % prefix: "printf",
- "@%s_quantile25" % prefix: "printf",
- "@%s_quantile50" % prefix: "printf",
- "@%s_quantile75" % prefix: "printf",
- "@%s_mu" % prefix: "printf"
- }
-
- # Data processing helper
- # (computes new distributions for sigma, s2, s10)
-
- def data_processing(self, dataframe):
-
- # Compute aggragated mu
- dataframe["mu"] = np.vectorize(
- np.average)(
- dataframe["mu"],
- weights=dataframe["nsamples"])
-
- # nsamples is the number of aggregated elements (as well as the number
- # of samples for our new sigma and s distributions)
- dataframe["nsamples"] = dataframe["nsamples"].apply(lambda x: len(x))
-
- dataframe["mu_x"] = dataframe.index
- # Make sure that strings don't excede a certain length
- dataframe["mu_x"] = dataframe["mu_x"].apply(
- lambda x: x[:17] + "[...]" + x[-17:] if len(x) > 39 else x
- )
-
- # Get quantiles and mu for sigma, s10, s2
- for prefix in ["sigma", "s10", "s2"]:
-
- dataframe["%s_x" % prefix] = dataframe["mu_x"]
-
- dataframe[prefix] = dataframe[prefix].apply(np.sort)
-
- dataframe["%s_min" % prefix] = dataframe[prefix].apply(np.min)
- dataframe["%s_quantile25" % prefix] = dataframe[prefix].apply(
- np.quantile, args=(0.25,))
- dataframe["%s_quantile50" % prefix] = dataframe[prefix].apply(
- np.quantile, args=(0.50,))
- dataframe["%s_quantile75" % prefix] = dataframe[prefix].apply(
- np.quantile, args=(0.75,))
- dataframe["%s_max" % prefix] = dataframe[prefix].apply(np.max)
- dataframe["%s_mu" % prefix] = dataframe[prefix].apply(np.average)
- del dataframe[prefix]
-
- return dataframe
-
- # Plots update function
-
- def update_plots(self):
-
- groupby_display = self.widgets["groupby_radio"].labels[
- self.widgets["groupby_radio"].active
- ]
- groupby = self.factors_dict[groupby_display]
-
- filterby_display = self.widgets["filterby_radio"].labels[
- self.widgets["filterby_radio"].active
- ]
- filterby = self.factors_dict[filterby_display]
-
- # Groupby and aggregate lines belonging to the same group in lists
-
- groups = self.run_data[
- self.run_data.index.isin(
- [self.widgets["select_filter"].value],
- level=filterby
- )
- ].groupby(groupby)
-
- groups = groups.agg({
- "sigma": lambda x: x.tolist(),
- "s10": lambda x: x.tolist(),
- "s2": lambda x: x.tolist(),
-
- "mu": lambda x: x.tolist(),
-
- # Used for mu weighted average first, then will be replaced
- "nsamples": lambda x: x.tolist()
- })
-
- # Compute the new distributions, ...
- groups = self.data_processing(groups).to_dict("list")
-
- # Update source
-
- # Assign each ColumnDataSource, starting with the boxplots
- for prefix in ["sigma", "s10", "s2"]:
-
- dict = {
- "%s_x" % prefix: groups["%s_x" % prefix],
- "%s_min" % prefix: groups["%s_min" % prefix],
- "%s_quantile25" % prefix: groups["%s_quantile25" % prefix],
- "%s_quantile50" % prefix: groups["%s_quantile50" % prefix],
- "%s_quantile75" % prefix: groups["%s_quantile75" % prefix],
- "%s_max" % prefix: groups["%s_max" % prefix],
- "%s_mu" % prefix: groups["%s_mu" % prefix],
-
- "nsamples": groups["nsamples"]
- }
-
- # Filter outliers if the box is checked
- if len(self.widgets["outliers_filtering_inspect"].active) > 0:
-
- # Boxplots will be filtered by max then min
- top_outliers = helper.detect_outliers(dict["%s_max" % prefix])
- helper.remove_boxplot_outliers(dict, top_outliers, prefix)
-
- bottom_outliers = helper.detect_outliers(
- dict["%s_min" % prefix])
- helper.remove_boxplot_outliers(dict, bottom_outliers, prefix)
-
- self.sources["%s_source" % prefix].data = dict
-
- # Finish with the mu plot
- dict = {
- "mu_x": groups["mu_x"],
- "mu": groups["mu"],
-
- "nsamples": groups["nsamples"]
- }
-
- self.sources["mu_source"].data = dict
-
- # Filter outliers if the box is checked
- if len(self.widgets["outliers_filtering_inspect"].active) > 0:
- mu_outliers = helper.detect_outliers(groups["mu"])
- groups["mu"] = helper.remove_outliers(groups["mu"], mu_outliers)
- groups["mu_x"] = helper.remove_outliers(
- groups["mu_x"], mu_outliers)
-
- # Update plots axis/titles
-
- # Get display string of the last (unselected) factor
- factors_dict = self.factors_dict.copy()
- del factors_dict[groupby_display]
- del factors_dict[filterby_display]
- for_all = list(factors_dict.keys())[0]
-
- # Update all display strings for plot title (remove caps, plural)
- groupby_display = groupby_display.lower()
- filterby_display = filterby_display.lower()[:-1]
- for_all = for_all.lower()
-
- self.plots["mu_inspect"].title.text = \
- "Empirical average μ of %s (groupped by %s, for all %s)" \
- % (filterby_display, groupby_display, for_all)
-
- self.plots["sigma_inspect"].title.text = \
- "Standard deviation σ of %s (groupped by %s, for all %s)" \
- % (filterby_display, groupby_display, for_all)
-
- self.plots["s10_inspect"].title.text = \
- "Significant digits s of %s (groupped by %s, for all %s)" \
- % (filterby_display, groupby_display, for_all)
-
- self.plots["s2_inspect"].title.text = \
- "Significant digits s of %s (groupped by %s, for all %s)" \
- % (filterby_display, groupby_display, for_all)
-
- helper.reset_x_range(self.plots["mu_inspect"], groups["mu_x"])
- helper.reset_x_range(self.plots["sigma_inspect"], groups["sigma_x"])
- helper.reset_x_range(self.plots["s10_inspect"], groups["s10_x"])
- helper.reset_x_range(self.plots["s2_inspect"], groups["s2_x"])
-
- # Widets' callback functions
-
- # Run selector callback
-
- def update_run(self, attrname, old, new):
-
- filterby = self.widgets["filterby_radio"].labels[
- self.widgets["filterby_radio"].active
- ]
- filterby = self.factors_dict[filterby]
-
- # Update run selection (by using dict mapping)
- self.current_run = self.runs_dict[new]
-
- # Update run data
- self.run_data = self.data[self.data["timestamp"] == self.current_run]
-
- # Save old selected option
- old_value = self.widgets["select_filter"].value
-
- # Update filter options
- options = self.run_data.index\
- .get_level_values(filterby).drop_duplicates().tolist()
- self.widgets["select_filter"].options = options
-
- if old_value not in self.widgets["select_filter"].options:
- self.widgets["select_filter"].value = options[0]
- # The update_var callback will be triggered by the assignment
-
- else:
- # Trigger the callback manually (since the plots need to be updated
- # anyway)
- self.update_filter("", "", old_value)
-
- # "Group by" radio
-
- def update_groupby(self, attrname, old, new):
-
- # Update "Filter by" radio list
- filterby_list = list(self.factors_dict.keys())
- del filterby_list[self.widgets["groupby_radio"].active]
- self.widgets["filterby_radio"].labels = filterby_list
-
- filterby = self.widgets["filterby_radio"].labels[
- self.widgets["filterby_radio"].active
- ]
- filterby = self.factors_dict[filterby]
-
- # Save old selected option
- old_value = self.widgets["select_filter"].value
-
- # Update filter options
- options = self.run_data.index\
- .get_level_values(filterby).drop_duplicates().tolist()
- self.widgets["select_filter"].options = options
-
- if old_value not in self.widgets["select_filter"].options:
- self.widgets["select_filter"].value = options[0]
- # The update_var callback will be triggered by the assignment
-
- else:
- # Trigger the callback manually (since the plots need to be updated
- # anyway)
- self.update_filter("", "", old_value)
-
- # "Filter by" radio
-
- def update_filterby(self, attrname, old, new):
-
- filterby = self.widgets["filterby_radio"].labels[
- self.widgets["filterby_radio"].active
- ]
- filterby = self.factors_dict[filterby]
-
- # Save old selected option
- old_value = self.widgets["select_filter"].value
-
- # Update filter selector options
- options = self.run_data.index\
- .get_level_values(filterby).drop_duplicates().tolist()
- self.widgets["select_filter"].options = options
-
- if old_value not in self.widgets["select_filter"].options:
- self.widgets["select_filter"].value = options[0]
- # The update_var callback will be triggered by the assignment
-
- else:
- # Trigger the callback manually (since the plots need to be updated
- # anyway)
- self.update_filter("", "", old_value)
-
- # Filter selector callback
-
- def update_filter(self, attrname, old, new):
- self.update_plots()
-
- # Filter outliers checkbox callback
-
- def update_outliers_filtering(self, attrname, old, new):
- # The status (checked/unchecked) of the checkbox is also verified inside
- # self.update_plots(), so calling this function is enough
- self.update_plots()
-
- # Bokeh setup functions
- # (for both variable and backend selection at once)
-
- def setup_plots(self):
-
- tools = "pan, wheel_zoom, xwheel_zoom, ywheel_zoom, reset, save"
-
- # Tooltips and formatters
-
- dotplot_tooltips = [
- ("Name", "@mu_x"),
- ("μ", "@mu{%0.18e}"),
- ("Number of samples (tests)", "@nsamples")
- ]
- dotplot_formatters = {
- "@mu": "printf"
- }
-
- sigma_boxplot_tooltips = self.gen_boxplot_tooltips("sigma")
- sigma_boxplot_tooltips_formatters = self.gen_boxplot_tooltips_formatters(
- "sigma")
-
- s10_boxplot_tooltips = self.gen_boxplot_tooltips("s10")
- s10_boxplot_tooltips_formatters = self.gen_boxplot_tooltips_formatters(
- "s10")
-
- s2_boxplot_tooltips = self.gen_boxplot_tooltips("s2")
- s2_boxplot_tooltips_formatters = self.gen_boxplot_tooltips_formatters(
- "s2")
-
- # Plots
-
- # Mu plot
- self.plots["mu_inspect"] = figure(
- name="mu_inspect",
- title="",
- plot_width=900, plot_height=400, x_range=[""],
- tools=tools, sizing_mode="scale_width"
- )
- plot.fill_dotplot(
- self.plots["mu_inspect"], self.sources["mu_source"], "mu",
- tooltips=dotplot_tooltips,
- tooltips_formatters=dotplot_formatters
- )
- self.doc.add_root(self.plots["mu_inspect"])
-
- # Sigma plot
- self.plots["sigma_inspect"] = figure(
- name="sigma_inspect",
- title="",
- plot_width=900, plot_height=400, x_range=[""],
- tools=tools, sizing_mode="scale_width"
- )
- plot.fill_boxplot(
- self.plots["sigma_inspect"],
- self.sources["sigma_source"],
- prefix="sigma",
- tooltips=sigma_boxplot_tooltips,
- tooltips_formatters=sigma_boxplot_tooltips_formatters)
- self.doc.add_root(self.plots["sigma_inspect"])
-
- # s plots
- self.plots["s10_inspect"] = figure(
- name="s10_inspect",
- title="",
- plot_width=900, plot_height=400, x_range=[""],
- tools=tools, sizing_mode='scale_width'
- )
- plot.fill_boxplot(
- self.plots["s10_inspect"],
- self.sources["s10_source"],
- prefix="s10",
- tooltips=s10_boxplot_tooltips,
- tooltips_formatters=s10_boxplot_tooltips_formatters)
- s10_tab_inspect = Panel(
- child=self.plots["s10_inspect"],
- title="Base 10")
-
- self.plots["s2_inspect"] = figure(
- name="s2_inspect",
- title="",
- plot_width=900, plot_height=400, x_range=[""],
- tools=tools, sizing_mode='scale_width'
- )
- plot.fill_boxplot(
- self.plots["s2_inspect"], self.sources["s2_source"], prefix="s2",
- tooltips=s2_boxplot_tooltips,
- tooltips_formatters=s2_boxplot_tooltips_formatters
- )
- s2_tab_inspect = Panel(child=self.plots["s2_inspect"], title="Base 2")
-
- s_tabs_inspect = Tabs(
- name="s_tabs_inspect",
- tabs=[s10_tab_inspect, s2_tab_inspect], tabs_location="below"
- )
- self.doc.add_root(s_tabs_inspect)
-
- def setup_widgets(self):
-
- # Generation of selectable items
-
- # Dict contains all inspectable runs (maps display strings to timestamps)
- # The dict structure allows to get the timestamp from the display string
- # in O(1)
- self.runs_dict = self.gen_runs_selection()
-
- # Dict maps display strings to column names for the different factors
- # (var, backend, test)
- self.factors_dict = {
- "Variables": "variable",
- "Backends": "vfc_backend",
- "Tests": "test"
- }
-
- # Run selection
-
- # Contains all options strings
- runs_display = list(self.runs_dict.keys())
- # Will be used when updating plots (contains actual number)
- self.current_run = self.runs_dict[runs_display[-1]]
- # Contains the selected option string, used to update current_n_runs
- current_run_display = runs_display[-1]
- # This contains only entries matching the run
- self.run_data = self.data[self.data["timestamp"] == self.current_run]
-
- change_run_callback_js = "updateRunMetadata(cb_obj.value);"
-
- self.widgets["select_run"] = Select(
- name="select_run", title="Run :",
- value=current_run_display, options=runs_display
- )
- self.doc.add_root(self.widgets["select_run"])
- self.widgets["select_run"].on_change("value", self.update_run)
- self.widgets["select_run"].js_on_change("value", CustomJS(
- code=change_run_callback_js,
- args=(dict(
- metadata=helper.metadata_to_dict(
- helper.get_metadata(self.metadata, self.current_run)
- )
- ))
- ))
-
- # Factors selection
-
- # "Group by" radio
- self.widgets["groupby_radio"] = RadioButtonGroup(
- name="groupby_radio",
- labels=list(self.factors_dict.keys()), active=0
- )
- self.doc.add_root(self.widgets["groupby_radio"])
- # The functions are defined inside the template to avoid writing too
- # much JS server side
- self.widgets["groupby_radio"].on_change(
- "active",
- self.update_groupby
- )
-
- # "Filter by" radio
- # Get all possible factors, and remove the one selected in "Group by"
- filterby_list = list(self.factors_dict.keys())
- del filterby_list[self.widgets["groupby_radio"].active]
-
- self.widgets["filterby_radio"] = RadioButtonGroup(
- name="filterby_radio",
- labels=filterby_list, active=0
- )
- self.doc.add_root(self.widgets["filterby_radio"])
- # The functions are defined inside the template to avoid writing too
- # much JS server side
- self.widgets["filterby_radio"].on_change(
- "active",
- self.update_filterby
- )
-
- # Filter selector
-
- filterby = self.widgets["filterby_radio"].labels[
- self.widgets["filterby_radio"].active
- ]
- filterby = self.factors_dict[filterby]
-
- options = self.run_data.index\
- .get_level_values(filterby).drop_duplicates().tolist()
-
- self.widgets["select_filter"] = Select(
- # We need a different name to avoid collision in the template with
- # the runs comparison's widget
- name="select_filter", title="Select a filter :",
- value=options[0], options=options
- )
- self.doc.add_root(self.widgets["select_filter"])
- self.widgets["select_filter"]\
- .on_change("value", self.update_filter)
-
- # Toggle for outliers filtering
-
- self.widgets["outliers_filtering_inspect"] = CheckboxGroup(
- name="outliers_filtering_inspect",
- labels=["Filter outliers"], active=[]
- )
- self.doc.add_root(self.widgets["outliers_filtering_inspect"])
- self.widgets["outliers_filtering_inspect"]\
- .on_change("active", self.update_outliers_filtering)
-
- # Communication methods
- # (to send/receive messages to/from master)
-
- # When received, switch to the run_name in parameter
-
- def switch_view(self, run_name):
- self.widgets["select_run"].value = run_name
-
- # Constructor
-
- def __init__(self, master, doc, data, metadata):
-
- self.master = master
-
- self.doc = doc
- self.data = data
- self.metadata = metadata
-
- self.sources = {
- "mu_source": ColumnDataSource(data={}),
- "sigma_source": ColumnDataSource(data={}),
- "s10_source": ColumnDataSource(data={}),
- "s2_source": ColumnDataSource(data={})
- }
-
- self.plots = {}
- self.widgets = {}
-
- # Setup Bokeh objects
- self.setup_plots()
- self.setup_widgets()
-
- # Pass the initial metadata to the template (will be updated in CustomJS
- # callbacks). This is required because metadata is not displayed in a
- # Bokeh widget, so we can't update this with a server callback.
- initial_run = helper.get_metadata(self.metadata, self.current_run)
- self.doc.template_variables["initial_timestamp"] = self.current_run
-
- # At this point, everything should have been initialized, so we can
- # show the plots for the first time
- self.update_plots()
diff --git a/ci/vfc_ci_report/main.py b/ci/vfc_ci_report/main.py
deleted file mode 100644
index 4af73d2..0000000
--- a/ci/vfc_ci_report/main.py
+++ /dev/null
@@ -1,208 +0,0 @@
-# Look for and read all the run files in the current directory (ending with
-# .vfcrunh5), and lanch a Bokeh server for the visualization of this data.
-
-import os
-import sys
-import time
-
-import pandas as pd
-
-from bokeh.plotting import curdoc
-
-# Local imports from vfc_ci_server
-import compare_runs
-import inspect_runs
-import helper
-
-##########################################################################
-
-# Read vfcrun files, and aggregate them in one dataset
-
-run_files = [f for f in os.listdir(".") if f.endswith(".vfcrun.h5")]
-
-if len(run_files) == 0:
- print(
- "Warning [vfc_ci]: Could not find any vfcrun files in the directory. "
- "This will result in server errors and prevent you from viewing the report.")
-
-# These are arrays of Pandas dataframes for now
-metadata = []
-data = []
-
-for f in run_files:
- metadata.append(pd.read_hdf(f, "metadata"))
- data.append(pd.read_hdf(f, "data"))
-
-metadata = pd.concat(metadata).sort_index()
-data = pd.concat(data).sort_index()
-
-
-# Generate the display strings for runs (runs ticks)
-# By doing this in master, we ensure the homogeneity of display strings
-# across all plots
-metadata["name"] = metadata.index.to_series().map(
- lambda x: helper.get_run_name(
- x,
- helper.get_metadata(metadata, x)["hash"]
- )
-)
-helper.reset_run_strings()
-
-metadata["date"] = metadata.index.to_series().map(
- lambda x: time.ctime(x)
-)
-
-
-##########################################################################
-
-
-curdoc().title = "Verificarlo Report"
-
-# Read server arguments
-# (this is quite easy because Bokeh server is called through a wrapper, so
-# we know exactly what the arguments might be)
-
-git_repo_linked = False
-commit_link = ""
-
-has_logo = False
-logo_url = ""
-
-for i in range(1, len(sys.argv)):
-
- # Look for the Git repository remote address
- # (if a Git repo is specified, the webpage will contain hyperlinks to the
- # repository and the different commits)
- if sys.argv[i] == "git":
- from urllib.parse import urlparse
-
- method = sys.argv[i + 1]
- address = sys.argv[i + 2]
- url = ""
-
- # Here, address is either the remote URL or the path to the local Git
- # repo (depending on the method)
-
- if method == "url":
- # We should directly have a Git URL
- url = address
-
- elif method == "directory":
- # Get the remote URL from the local repo
- from git import Repo
- repo = Repo(address)
- url = repo.remotes.origin.url
-
- else:
- raise ValueError(
- "Error [vfc_ci]: The specified method to get the Git "
- "repository is invalid. Are you calling Bokeh directly "
- "instead of using the Verificarlo wrapper ?"
- )
-
- # At this point, "url" should be set correctly, we can get the repo's
- # URL and name, after making sure we're on a Git URL
-
- parsed_url = urlparse(url)
-
- path = parsed_url.path.split("/")
- if len(path) < 3:
- raise ValueError(
- "Error [vfc_ci]: The found URL doesn't seem to be pointing "
- "to a Git repository (path is too short)"
- )
-
- repo_name = path[2]
-
- curdoc().template_variables["repo_url"] = url
- curdoc().template_variables["repo_name"] = repo_name
-
- # We should have a "github.com" or a "*gitlab*" URL
-
- if parsed_url.netloc == "github.com":
- commit_link = "https://%s%s/commit/" \
- % (parsed_url.netloc, parsed_url.path)
-
- curdoc().template_variables["commit_link"] = commit_link
- curdoc().template_variables["git_host"] = "GitHub"
-
- # Used in Bokeh tooltips
- commit_link = commit_link + "@hash"
-
- # We assume we have a GitLab URL
- else:
- commit_link = "https://%s%s/-/commit/" \
- % (parsed_url.netloc, parsed_url.path)
-
- curdoc().template_variables["commit_link"] = commit_link
- curdoc().template_variables["git_host"] = "GitLab"
-
- # Used in Bokeh tooltips
- commit_link = commit_link + "@hash"
-
- git_repo_linked = True
-
- # Look for a logo URL
- # If a logo URL is specified, it will be included in the report's header
- if sys.argv[i] == "logo":
- curdoc().template_variables["logo_url"] = sys.argv[i + 1]
- has_logo = True
-
-
-# After the loop, we know if a repo has been linked, if we have a logo, ...
-curdoc().template_variables["git_repo_linked"] = git_repo_linked
-curdoc().template_variables["has_logo"] = has_logo
-
-
-##########################################################################
-
-# Setup report views
-
-# Define a ViewsMaster class to allow two-ways communication between views.
-# This approach by classes allows us to have separate scopes for each view and
-# will be useful if we want to add new views at some point in the future
-# (instead of having n views with n-1 references each).
-
-class ViewsMaster:
-
- # Communication functions
-
- def go_to_inspect(self, run_name):
- self.inspect.switch_view(run_name)
-
- # Constructor
-
- def __init__(self, data, metadata, git_repo_linked, commit_link):
-
- self.data = data
- self.metadata = metadata
- self.git_repo_linked = git_repo_linked
- self.commit_link = commit_link
-
- # Pass metadata to the template as a JSON string
- curdoc().template_variables["metadata"] = self.metadata.to_json(
- orient="index")
-
- # Runs comparison
- self.compare = compare_runs.CompareRuns(
- master=self,
- doc=curdoc(),
- data=data,
- metadata=metadata,
- )
-
- # Runs inspection
- self.inspect = inspect_runs.InspectRuns(
- master=self,
- doc=curdoc(),
- data=data,
- metadata=metadata,
- )
-
-
-views_master = ViewsMaster(
- data=data,
- metadata=metadata,
- git_repo_linked=git_repo_linked,
- commit_link=commit_link
-)
diff --git a/ci/vfc_ci_report/plot.py b/ci/vfc_ci_report/plot.py
deleted file mode 100644
index f4bb20f..0000000
--- a/ci/vfc_ci_report/plot.py
+++ /dev/null
@@ -1,142 +0,0 @@
-# General functions for filling plots with data in all report's views
-
-from bokeh.plotting import figure
-from bokeh.models import HoverTool, TapTool, CustomJS
-
-from math import pi
-
-
-def fill_dotplot(
- plot, source, data_field,
- tooltips=None, tooltips_formatters=None,
- js_tap_callback=None, server_tap_callback=None,
- lines=False,
- lower_bound=False
-):
-
- # (Optional) Tooltip and tooltip formatters
- if tooltips is not None:
- hover = HoverTool(tooltips=tooltips, mode="vline", names=["circle"])
-
- if tooltips_formatters is not None:
- hover.formatters = tooltips_formatters
-
- plot.add_tools(hover)
-
- # (Optional) Add TapTool (for JS tap callback)
- if js_tap_callback is not None:
- tap = TapTool(callback=CustomJS(code=js_tap_callback))
- plot.add_tools(tap)
-
- # (Optional) Add segment to represent a lower bound
- if lower_bound:
- lower_segment = plot.segment(
- x0="%s_x" % data_field, y0=data_field,
- x1="%s_x" % data_field, y1="%s_lower_bound" % data_field,
- source=source, line_color="black"
- )
-
- # Draw dots (actually Bokeh circles)
- circle = plot.circle(
- name="circle",
- x="%s_x" % data_field, y=data_field, source=source, size=12
- )
-
- # (Optional) Draw lines between dots
- if lines:
- line = plot.line(x="%s_x" % data_field, y=data_field, source=source)
-
- # (Optional) Add server tap callback
- if server_tap_callback is not None:
- circle.data_source.selected.on_change("indices", server_tap_callback)
-
- # Plot appearance
- plot.xgrid.grid_line_color = None
- plot.ygrid.grid_line_color = None
-
- plot.yaxis[0].formatter.power_limit_high = 0
- plot.yaxis[0].formatter.power_limit_low = 0
- plot.yaxis[0].formatter.precision = 3
-
- plot.xaxis[0].major_label_orientation = pi / 8
-
-
-def fill_boxplot(
- plot, source,
- prefix="",
- tooltips=None, tooltips_formatters=None,
- js_tap_callback=None, server_tap_callback=None
-):
-
- # (Optional) Tooltip and tooltip formatters
- if tooltips is not None:
- hover = HoverTool(tooltips=tooltips, mode="vline", names=["full_box"])
-
- if tooltips_formatters is not None:
- hover.formatters = tooltips_formatters
-
- plot.add_tools(hover)
-
- # (Optional) Add TapTool (for JS tap callback)
- if js_tap_callback is not None:
- tap = TapTool(callback=CustomJS(code=js_tap_callback))
- plot.add_tools(tap)
-
- # Draw boxes (the prefix argument modifies the fields of ColumnDataSource
- # that are used)
-
- if prefix != "":
- prefix = "%s_" % prefix
-
- # Stems
- top_stem = plot.segment(
- x0="%sx" % prefix, y0="%smax" % prefix,
- x1="%sx" % prefix, y1="%squantile75" % prefix,
- source=source, line_color="black"
- )
- bottom_stem = plot.segment(
- x0="%sx" % prefix, y0="%smin" % prefix,
- x1="%sx" % prefix, y1="%squantile25" % prefix,
- source=source, line_color="black"
- )
-
- # Boxes
- full_box = plot.vbar(
- name="full_box",
- x="%sx" % prefix, width=0.5,
- top="%squantile75" % prefix, bottom="%squantile25" % prefix,
- source=source, line_color="black"
- )
- bottom_box = plot.vbar(
- x="%sx" % prefix, width=0.5,
- top="%squantile50" % prefix, bottom="%squantile25" % prefix,
- source=source, line_color="black"
- )
-
- # Mu dot
- mu_dot = plot.dot(
- x="%sx" % prefix, y="%smu" % prefix, size=30, source=source,
- color="black"
- )
-
- # (Optional) Add server tap callback
- if server_tap_callback is not None:
- top_stem.data_source.selected.on_change("indices", server_tap_callback)
- bottom_stem.data_source.selected.on_change(
- "indices", server_tap_callback)
-
- full_box.data_source.selected.on_change("indices", server_tap_callback)
- bottom_box.data_source.selected.on_change(
- "indices", server_tap_callback)
-
- mu_dot.data_source.selected.on_change("indices", server_tap_callback)
-
- # Plot appearance
- plot.xgrid.grid_line_color = None
- plot.ygrid.grid_line_color = None
-
- plot.yaxis[0].formatter.power_limit_high = 0
- plot.yaxis[0].formatter.power_limit_low = 0
- plot.yaxis[0].formatter.precision = 3
-
- plot.xaxis[0].major_label_orientation = pi / 8
diff --git a/ci/vfc_ci_report/templates/index.html b/ci/vfc_ci_report/templates/index.html
deleted file mode 100644
index 8fea8ec..0000000
--- a/ci/vfc_ci_report/templates/index.html
+++ /dev/null
@@ -1,482 +0,0 @@
-
-
-
-
- Verificarlo Report
-
-
-
-
-
-
-
-
-
- {% extends base %}
-
-
-
-
- {% block contents %}
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
Selectors
-
- {{ embed(roots.test_filter) }}
- {{ embed(roots.select_test) }}
-
- {{ embed(roots.select_var) }}
-
- {{ embed(roots.select_backend) }}
-
- {{ embed(roots.outliers_filtering_compare) }}
-
-
- {{ embed(roots.select_n_runs) }}
-
-
-
-
-
- Tip : You can click on any element of the plots
- to inspect the corresponding run in details.
-
-
-
-
-
-
-
-
-
Plots
-
-
- {{ embed(roots.s_tabs) }}
-
-
-
-
-
- {{ embed(roots.sigma_plot) }}
-
-
-
-
-
- {{ embed(roots.boxplot) }}
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
Selectors
- {{ embed(roots.select_run) }}
-
-
-
- Group by :
- {{ embed(roots.groupby_radio) }}
-
-
-
- Filter by :
- {{ embed(roots.filterby_radio) }}
- {{ embed(roots.select_filter) }}
-
-
-
- {{ embed(roots.outliers_filtering_inspect) }}
-
-
-
-
-
Run metadata
-
-
Date :
-
-
-
-
-
-
-
-
-
- This run is not linked to a Git commit.
-
-
-
-
-
-
-
-
-
Plots
-
- {{ embed(roots.s_tabs_inspect) }}
-
-
-
-
-
- {{ embed(roots.sigma_inspect) }}
-
-
-
-
-
- {{ embed(roots.mu_inspect) }}
-
-
-
-
-
-
-
-
-
-
-
-
- {% if has_logo %}
-
- {% else %}
-
- {% endif %}
-
-
-
-
-
-
-
- {% endblock %}
-
-
-
diff --git a/ci/workflow_templates/ci_README.j2.md b/ci/workflow_templates/ci_README.j2.md
deleted file mode 100644
index a41ca96..0000000
--- a/ci/workflow_templates/ci_README.j2.md
+++ /dev/null
@@ -1,7 +0,0 @@
-## Verificarlo CI : `{{dev_branch}}`
-
-You are on the `{{ci_branch}}` branch, which is automatically updated with the
-[Verificarlo](https://github.com/verificarlo/verificarlo) test results from
-`{{dev_branch}}` (in the `vfcruns` directory).
-
-You can start a Verificarlo CI server at anytime using the run files of this branch.
diff --git a/ci/workflow_templates/gitlab-ci.j2.yml b/ci/workflow_templates/gitlab-ci.j2.yml
deleted file mode 100644
index de26aeb..0000000
--- a/ci/workflow_templates/gitlab-ci.j2.yml
+++ /dev/null
@@ -1,42 +0,0 @@
-# This workflow will be executed when {{dev_branch}} is updated:
-# it will run the configured tests and upload the results on vfc_ci_master.
-
-image: verificarlo/verificarlo
-
-
-stages:
- - run_verificarlo_tests
-
-
-run_verificarlo_tests:
- stage: run_verificarlo_tests
-
- before_script:
- - git remote set-url origin https://{{username}}:${CI_PUSH_TOKEN}@{{remote_url}}.git
- - git config --global user.email "{{email}}"
- - git config --global user.name "{{username}}"
-
- script:
- # We will probably drop these installations when integrating CI into
- # Verificarlo
- - pip install numpy scipy pandas bokeh jinja2 tables GitPython
- - apt update
- - apt install wget
- - wget https://raw.githubusercontent.com/verificarlo/significantdigits/main/sigdigits.py -P /usr/local/lib/python3.8/dist-packages
-
- - ./vfc_ci test -g -r
- - git_hash=$(git rev-parse --short "$CI_COMMIT_SHA")
- - git fetch --all
- - git checkout -b {{ci_branch}} origin/{{ci_branch}}
- - mkdir -p vfcruns
- - mv *.vfcrun.h5 vfcruns
- - git add vfcruns/*
- - git commit -m "[auto] New test results for commit ${git_hash}"
- - git push
-
- rules:
- - if: '$CI_COMMIT_BRANCH == "{{dev_branch}}"'
-
- artifacts:
- paths:
- - "*.vfcraw.h5"
diff --git a/ci/workflow_templates/vfc_test_workflow.j2.yml b/ci/workflow_templates/vfc_test_workflow.j2.yml
deleted file mode 100644
index b39e21e..0000000
--- a/ci/workflow_templates/vfc_test_workflow.j2.yml
+++ /dev/null
@@ -1,57 +0,0 @@
-# This workflow will be executed when {{dev_branch}} is updated:
-# it will run the configured tests and upload the results on {{ci_branch}}.
-
-name: "Verificarlo CI ({{dev_branch}})"
-
-on:
- # Triggers the workflow when {{dev_branch}} is updated
- push:
- branches: [ {{dev_branch}} ]
-
- workflow_dispatch:
-
-
-jobs:
- run_verificarlo_tests:
- runs-on: ubuntu-latest
- container: verificarlo/verificarlo
-
- steps:
- - uses: actions/checkout@v2
- with:
- fetch-depth: 0
-
- # We will probably drop these installations when integrating CI into
- # Verificarlo
- - name: Install Python requirements
- run: |
- pip install numpy scipy pandas bokeh jinja2 tables GitPython
- apt update
- apt install wget
- wget https://raw.githubusercontent.com/verificarlo/significantdigits/main/sigdigits.py -P /usr/local/lib/python3.8/dist-packages
-
- - name: Run tests
- # We assume the script is included in the repo for now
- # (we'll probably want to remove "./" if the script ends up being integrated
- # in Verificarlo and becomes available system-wide)
- run: ./vfc_ci test -g -r
-
- - name: Commit test results
- run: |
- git_hash=$(git rev-parse --short "$GITHUB_SHA")
-
- git config --local user.email "action@github.com"
- git config --local user.name "GitHub Action"
-
- git checkout {{ci_branch}}
- mkdir -p vfcruns
- mv *.vfcrun.h5 vfcruns
- git add vfcruns/*
- git commit -m "[auto] New test results for commit ${git_hash}"
- git push
-
- - name: Upload raw results as artifacts
- uses: actions/upload-artifact@v2
- with:
- {% raw %}name: ${{github.sha}}.vfcraw{% endraw %}
- path: ./*.vfcraw.h5
diff --git a/tests/vfc_test_h5.cpp b/tests/vfc_test_h5.cpp
index d7f3c8b..e567b9f 100644
--- a/tests/vfc_test_h5.cpp
+++ b/tests/vfc_test_h5.cpp
@@ -150,26 +150,33 @@ int test_cycle(H5File file, int cycle, std::string version,
}
int main(int argc, char **argv) {
- if (argc != 3) {
+ if (argc != 2) {
std::cerr << "Execute from within '/'" << std::endl;
- std::cerr << "usage: test_h5 " << std::endl;
+ std::cerr << "usage: vfc_test_h5 " << std::endl;
return 1;
}
- std::string version(argv[1]);
- std::vector cycles_list = get_cycles_list(argv[2]);
+ std::vector cycles_list = get_cycles_list(argv[1]);
H5File file(FILE_NAME, H5F_ACC_RDONLY);
vfc_probes probes = vfc_init_probes();
probes = vfc_init_probes();
+ std::vector versions = {"maponia3", "sm1", "sm2", "sm3", "sm4"};
+
bool ok;
for (int i = 0; i < cycles_list.size(); i++) {
- ok = test_cycle(file, cycles_list[i], version, &probes);
- if (ok) {
- std::cout << "ok -- cycle " << std::to_string(i) << std::endl;
- } else {
- std::cerr << "failed -- cycle " << std::to_string(i) << std::endl;
+
+ for(const auto& version: versions) {
+
+ ok = test_cycle(file, cycles_list[i], version, &probes);
+ if (ok) {
+ std::cout << "ok -- cycle " << std::to_string(i) << std::endl;
+ } else {
+ std::cerr << "failed -- cycle " << std::to_string(i) << std::endl;
+ }
+
}
+
}
vfc_dump_probes(&probes);
diff --git a/vfc_ci b/vfc_ci
deleted file mode 100755
index be0afdb..0000000
--- a/vfc_ci
+++ /dev/null
@@ -1,199 +0,0 @@
-#!/usr/bin/env python3
-
-# This is the entry point of the Verificarlo CI command line interface, which is
-# based on argparse and this article :
-# https://mike.depalatis.net/blog/simplifying-argparse.html
-# From here, 3 subcommands can be called :
-# - setup : create a vfc_ci branch and workflow on the current Git repo
-# - test : run and export test results according to the vfc_tests_config.json
-# - serve : launch a Bokeh server to visualize run results
-
-import argparse
-
-##########################################################################
-
-# Parameters validation helpers
-
-
-def is_port(string):
- value = int(string)
- if value < 0 or value > 65535:
- raise argparse.ArgumentTypeError("Value has to be between 0 and 65535")
- return value
-
-
-def is_directory(string):
- import os
-
- isdir = os.path.isdir(string)
- if not isdir:
- raise argparse.ArgumentTypeError("Directory does not exist")
-
- return string
-
-
-##########################################################################
-
- # Subcommand decorator
-
-cli = argparse.ArgumentParser(
- description="Define, run, automatize, and visualize custom Verificarlo tests."
-)
-subparsers = cli.add_subparsers(dest="subcommand")
-
-
-def subcommand(description="", args=[], parent=subparsers):
- def decorator(func):
- parser = parent.add_parser(func.__name__, description=description)
- for arg in args:
- parser.add_argument(*arg[0], **arg[1])
- parser.set_defaults(func=func)
- return decorator
-
-
-def argument(*name_or_flags, **kwargs):
- return ([*name_or_flags], kwargs)
-
-
-##########################################################################
-
- # "setup" subcommand
-
-@subcommand(
- description="Create an automated workflow to execute Verificarlo tests.",
- args=[
- argument(
- "git_host",
- help="""
- specify where your repository is hosted
- """,
- choices=["github", "gitlab"]
- )
- ]
-)
-def setup(args):
- import ci.setup
- ci.setup.setup(args.git_host)
-
- # "test" subcommand
-
-
-@subcommand(
- description="Execute predefined Verificarlo tests and save their results.",
- args=[
- argument(
- "-g", "--is-git-commit",
- help="""
- When specified, the last Git commit of the local repository (working
- directory) will be fetched and associated with the run.
- """,
- action="store_true"
- ),
- argument(
- "-r", "--export-raw-results",
- help="""
- Specify if an additional HDF5 file containing the raw results must be
- exported.
- """,
- action="store_true"
- ),
- argument(
- "-d", "--dry-run",
- help="""
- Perform a dry run by not saving the test results.
- """,
- action="store_true"
- )
- ]
-)
-def test(args):
- import ci.test
- ci.test.run(args.is_git_commit, args.export_raw_results, args.dry_run)
-
- # "serve" subcommand
-
-
-@subcommand(
- description="""
- Start a server to visualize Verificarlo test results.
- """,
- args=[
- argument(
- "-s", "--show",
- help="""
- Specify if the report must be opened in the browser at server
- startup.
- """,
- action="store_true"
- ),
- argument(
- "-gd", "--git-directory",
- help="""
- Path to a local Git repository. The report will be linked to the
- remote URL (GitHub and GitLab are supported).
- """,
- type=is_directory
- ),
- argument(
- "-gu", "--git-url",
- help="""
- GitHub or GitLab repository URL. The report will be linked to this
- URL.
- """,
- type=str
- ),
- argument(
- "-p", "--port",
- help="""
- The port on which the server will run. Defaults to 8080.
- """,
- type=is_port,
- default=8080
- ),
- argument(
- "-a", "--allow-origin",
- help="""
- The origin (URL) from which the report will be accessible.
- Port number must not be specified. Defaults to * (allow everything).
- """,
- type=str,
- default="*"
- ),
- argument(
- "-l", "--logo",
- help="""
- Specify the URL of an image to be displayed in the report header.
- """,
- type=str
- )
- ]
-)
-def serve(args):
-
- # git_directory and git_url are supposed to be exclusive
- if args.git_directory is not None and args.git_url is not None:
- raise argparse.ArgumentTypeError(
- "\"-gd\" / \"--git-directory\" and \"-gu\" / \"--git-url\" are "
- "mutually exclusive. Please make sure to use at most one of them."
- )
-
- import ci.serve
- ci.serve.serve(
- args.show,
- args.git_directory,
- args.git_url,
- args.port,
- args.allow_origin,
- args.logo
- )
-
-
-###############################################################################
-
- # Main command group and entry point
-if __name__ == "__main__":
- args = cli.parse_args()
- if args.subcommand is None:
- cli.print_help()
- else:
- args.func(args)
diff --git a/vfc_tests_config.json b/vfc_tests_config.json
index 5c61a1e..c5119a0 100644
--- a/vfc_tests_config.json
+++ b/vfc_tests_config.json
@@ -1,69 +1,10 @@
+
{
"make_command": "make -f Makefile.verificarlo",
"executables": [
{
"executable": "bin/vfc_test_h5",
- "parameters" : "maponia3 vfc_ci_cycles.txt",
- "vfc_backends": [
- {
- "name": "libinterflop_mca.so",
- "repetitions": 50
- },
- {
- "name": "libinterflop_mca.so --mode=rr",
- "repetitions": 50
- }
- ]
- },
-
- {
- "executable": "bin/vfc_test_h5",
- "parameters" : "sm1 vfc_ci_cycles.txt",
- "vfc_backends": [
- {
- "name": "libinterflop_mca.so",
- "repetitions": 50
- },
- {
- "name": "libinterflop_mca.so --mode=rr",
- "repetitions": 50
- }
- ]
- },
-
- {
- "executable": "bin/vfc_test_h5",
- "parameters" : "sm2 vfc_ci_cycles.txt",
- "vfc_backends": [
- {
- "name": "libinterflop_mca.so",
- "repetitions": 50
- },
- {
- "name": "libinterflop_mca.so --mode=rr",
- "repetitions": 50
- }
- ]
- },
-
- {
- "executable": "bin/vfc_test_h5",
- "parameters" : "sm3 vfc_ci_cycles.txt",
- "vfc_backends": [
- {
- "name": "libinterflop_mca.so",
- "repetitions": 50
- },
- {
- "name": "libinterflop_mca.so --mode=rr",
- "repetitions": 50
- }
- ]
- },
-
- {
- "executable": "bin/vfc_test_h5",
- "parameters" : "sm4 vfc_ci_cycles.txt",
+ "parameters" : "vfc_ci_cycles.txt",
"vfc_backends": [
{
"name": "libinterflop_mca.so",