mirror of
https://github.com/TREX-CoE/Sherman-Morrison.git
synced 2024-12-25 05:43:54 +01:00
Remove source code of vfc_ci and improve vfc_test_h5.cpp
Since vfc_ci has been merged wih the main verificarlo repository and is now available in Verificarlo's Docker, the source code of vfc_ci can now be removed. Also, vfc_test_h5.cpp has been modified so that every algorithm is called directly. This results in a much shorter vfc_tests_config.json since we don't need to vall vfc_test_h5 with all algorithms as arguments.
This commit is contained in:
Aurélien Delval
parent
cb8cb2c426
commit
0114fd7b14
15
.github/workflows/vfc_test_workflow.yml
vendored
15
.github/workflows/vfc_test_workflow.yml
vendored
@ -21,35 +21,24 @@ jobs:
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
# We will probably drop these installations when integrating CI into
|
||||
# Verificarlo
|
||||
- name: Install Python requirements
|
||||
- name: Install HDF5 requirements
|
||||
run: |
|
||||
pip install numpy scipy pandas bokeh jinja2 tables GitPython
|
||||
apt update
|
||||
apt install -y wget libhdf5-dev g++
|
||||
wget https://raw.githubusercontent.com/verificarlo/significantdigits/main/sigdigits.py -P /usr/local/lib/python3.8/dist-packages
|
||||
|
||||
- name: Run tests
|
||||
# We assume the script is included in the repo for now
|
||||
# (we'll probably want to remove "./" if the script ends up being integrated
|
||||
# in Verificarlo and becomes available system-wide)
|
||||
run: ./vfc_ci test -g -r
|
||||
run: vfc_ci test -g -r
|
||||
|
||||
- name: Commit test results
|
||||
run: |
|
||||
git_hash=$(git rev-parse --short "$GITHUB_SHA")
|
||||
|
||||
git config --local user.email "action@github.com"
|
||||
git config --local user.name "GitHub Action"
|
||||
|
||||
git checkout vfc_ci_dev
|
||||
mkdir -p vfcruns
|
||||
mv *.vfcrun.h5 vfcruns
|
||||
git add vfcruns/*
|
||||
git commit -m "[auto] New test results for commit ${git_hash}"
|
||||
git push
|
||||
|
||||
- name: Upload raw results as artifacts
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
|
||||
@ -20,7 +20,7 @@ ifeq ($(MKL),-DMKL)
|
||||
ifeq ($(ENV),INTEL)
|
||||
LFLAGS = -mkl=sequential # implicit
|
||||
else
|
||||
LFLAGS = $(H5LFLAGS)
|
||||
LFLAGS = $(H5LFLAGS) -lvfc_probes
|
||||
endif
|
||||
endif
|
||||
H5CXXFLAGS = $(CXXFLAGS)
|
||||
|
||||
28
ci/serve.py
28
ci/serve.py
@ -1,28 +0,0 @@
|
||||
# Server for the Verificarlo CI report. This is simply a wrapper to avoid
|
||||
# calling Bokeh directly.
|
||||
|
||||
import os
|
||||
|
||||
|
||||
def serve(show, git_directory, git_url, port, allow_origin, logo_url):
|
||||
|
||||
# Prepare arguments
|
||||
show = "--show" if show else ""
|
||||
|
||||
git = ""
|
||||
if git_directory is not None:
|
||||
git = "git directory %s" % git_directory
|
||||
if git_url is not None:
|
||||
git = "git url %s" % git_url
|
||||
|
||||
logo = ""
|
||||
if logo_url is not None:
|
||||
logo = "logo %s" % logo_url
|
||||
|
||||
dirname = os.path.dirname(__file__)
|
||||
|
||||
# Call the "bokeh serve" command on the system
|
||||
command = "bokeh serve %s/vfc_ci_report %s --allow-websocket-origin=%s:%s --port %s --args %s %s" \
|
||||
% (dirname, show, allow_origin, port, port, git, logo)
|
||||
|
||||
os.system(command)
|
||||
146
ci/setup.py
146
ci/setup.py
@ -1,146 +0,0 @@
|
||||
# Helper script to set up Verificarlo CI on the current branch
|
||||
|
||||
import git
|
||||
|
||||
import sys
|
||||
import os
|
||||
from jinja2 import Environment, FileSystemLoader
|
||||
|
||||
##########################################################################
|
||||
|
||||
# Helper functions
|
||||
|
||||
|
||||
def gen_readme(dev_branch, ci_branch):
|
||||
|
||||
# Init template loader
|
||||
path = os.path.dirname(os.path.abspath(__file__))
|
||||
env = Environment(loader=FileSystemLoader(path))
|
||||
template = env.get_template("workflow_templates/ci_README.j2.md")
|
||||
|
||||
# Render template
|
||||
render = template.render(dev_branch=dev_branch, ci_branch=ci_branch)
|
||||
|
||||
# Write template
|
||||
with open("README.md", "w") as fh:
|
||||
fh.write(render)
|
||||
|
||||
|
||||
def gen_workflow(git_host, dev_branch, ci_branch, repo):
|
||||
|
||||
# Init template loader
|
||||
path = os.path.dirname(os.path.abspath(__file__))
|
||||
env = Environment(loader=FileSystemLoader(path))
|
||||
|
||||
if git_host == "github":
|
||||
# Load template
|
||||
template = env.get_template(
|
||||
"workflow_templates/vfc_test_workflow.j2.yml")
|
||||
|
||||
# Render it
|
||||
render = template.render(dev_branch=dev_branch, ci_branch=ci_branch)
|
||||
|
||||
# Write the file
|
||||
filename = ".github/workflows/vfc_test_workflow.yml"
|
||||
os.makedirs(os.path.dirname(filename), exist_ok=True)
|
||||
with open(filename, "w") as fh:
|
||||
fh.write(render)
|
||||
|
||||
if git_host == "gitlab":
|
||||
template = env.get_template("workflow_templates/gitlab-ci.j2.yml")
|
||||
|
||||
# Ask for the user who will run the jobs (Gitlab specific)
|
||||
username = input(
|
||||
"[vfc_ci] Enter the name of the user who will run the CI jobs:")
|
||||
email = input(
|
||||
"[vfc_ci] Enter the e-mail of the user who will run the CI jobs:")
|
||||
|
||||
remote_url = repo.remotes[0].config_reader.get("url")
|
||||
remote_url = remote_url.replace("http://", "")
|
||||
remote_url = remote_url.replace("https://", "")
|
||||
|
||||
render = template.render(
|
||||
dev_branch=dev_branch,
|
||||
ci_branch=ci_branch,
|
||||
username=username,
|
||||
email=email,
|
||||
remote_url=remote_url
|
||||
)
|
||||
|
||||
filename = ".gitlab-ci.yml"
|
||||
with open(filename, "w") as fh:
|
||||
fh.write(render)
|
||||
|
||||
|
||||
##########################################################################
|
||||
|
||||
def setup(git_host):
|
||||
|
||||
# Init repo and make sure that the workflow setup is possible
|
||||
|
||||
repo = git.Repo(".")
|
||||
repo.remotes.origin.fetch()
|
||||
|
||||
# Make sure that repository is clean
|
||||
assert(not repo.is_dirty()), "Error [vfc_ci]: Unstaged changes detected " \
|
||||
"in your work tree."
|
||||
|
||||
dev_branch = repo.active_branch
|
||||
dev_branch_name = str(dev_branch)
|
||||
dev_remote = dev_branch.tracking_branch()
|
||||
|
||||
# Make sure that the active branch (on which to setup the workflow) has a
|
||||
# remote
|
||||
assert(dev_remote is not None), "Error [vfc_ci]: The current branch doesn't " \
|
||||
"have a remote."
|
||||
|
||||
# Make sure that we are not behind the remote (so we can push safely later)
|
||||
rev = "%s...%s" % (dev_branch_name, str(dev_remote))
|
||||
commits_behind = list(repo.iter_commits(rev))
|
||||
assert(commits_behind == []), "Error [vfc_ci]: The local branch seems " \
|
||||
"to be at least one commit behind remote."
|
||||
|
||||
# Commit the workflow on the current (dev) branch
|
||||
|
||||
ci_branch_name = "vfc_ci_%s" % dev_branch_name
|
||||
gen_workflow(git_host, dev_branch_name, ci_branch_name, repo)
|
||||
repo.git.add(".")
|
||||
repo.index.commit("[auto] Set up Verificarlo CI on this branch")
|
||||
repo.remote(name="origin").push()
|
||||
|
||||
# Create the CI branch (orphan branch with a readme on it)
|
||||
# (see : https://github.com/gitpython-developers/GitPython/issues/615)
|
||||
|
||||
repo.head.reference = git.Head(repo, "refs/heads/" + ci_branch_name)
|
||||
|
||||
repo.index.remove(["*"])
|
||||
gen_readme(dev_branch_name, ci_branch_name)
|
||||
repo.index.add(["README.md"])
|
||||
|
||||
repo.index.commit(
|
||||
"[auto] Create the Verificarlo CI branch for %s" % dev_branch_name,
|
||||
parent_commits=None
|
||||
)
|
||||
repo.remote(name="origin").push(
|
||||
refspec="%s:%s" % (ci_branch_name, ci_branch_name)
|
||||
)
|
||||
|
||||
# Force checkout back to the original (dev) branch
|
||||
repo.git.checkout(dev_branch_name, force=True)
|
||||
|
||||
# Print termination messages
|
||||
|
||||
print(
|
||||
"Info [vfc_ci]: A Verificarlo CI workflow has been setup on "
|
||||
"%s." % dev_branch_name
|
||||
)
|
||||
print(
|
||||
"Info [vfc_ci]: Make sure that you have a \"vfc_tests_config.json\" on "
|
||||
"this branch. You can also perform a \"vfc_ci test\" dry run before "
|
||||
"pushing other commits.")
|
||||
|
||||
if git_host == "gitlab":
|
||||
print(
|
||||
"Info [vfc_ci]: Since you are using GitLab, make sure that you "
|
||||
"have created an access token for the user you specified (registered "
|
||||
"as a variable called \"CI_PUSH_TOKEN\" in your repository).")
|
||||
367
ci/test.py
367
ci/test.py
@ -1,367 +0,0 @@
|
||||
# This script reads the vfc_tests_config.json file and executes tests accordingly
|
||||
# It will also generate a ... .vfcrunh5 file with the results of the run
|
||||
|
||||
import sigdigits as sd
|
||||
import scipy.stats
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import os
|
||||
import json
|
||||
|
||||
import calendar
|
||||
import time
|
||||
|
||||
# Forcing an older pickle protocol allows backwards compatibility when reading
|
||||
# HDF5 written in 3.8+ using an older version of Python
|
||||
import pickle
|
||||
pickle.HIGHEST_PROTOCOL = 4
|
||||
|
||||
|
||||
# Magic numbers
|
||||
min_pvalue = 0.05
|
||||
max_zscore = 3
|
||||
|
||||
|
||||
##########################################################################
|
||||
|
||||
# Helper functions
|
||||
|
||||
# Read a CSV file outputted by vfc_probe as a Pandas dataframe
|
||||
def read_probes_csv(filepath, backend, warnings, execution_data):
|
||||
|
||||
try:
|
||||
results = pd.read_csv(filepath)
|
||||
|
||||
except FileNotFoundError:
|
||||
print(
|
||||
"Warning [vfc_ci]: Probes not found, your code might have crashed "
|
||||
"or you might have forgotten to call vfc_dump_probes"
|
||||
)
|
||||
warnings.append(execution_data)
|
||||
return pd.DataFrame(
|
||||
columns=["test", "variable", "values", "vfc_backend"]
|
||||
)
|
||||
|
||||
except Exception:
|
||||
print(
|
||||
"Warning [vfc_ci]: Your probes could not be read for some unknown "
|
||||
"reason"
|
||||
)
|
||||
warnings.append(execution_data)
|
||||
return pd.DataFrame(
|
||||
columns=["test", "variable", "values", "vfc_backend"]
|
||||
)
|
||||
|
||||
if len(results) == 0:
|
||||
print(
|
||||
"Warning [vfc_ci]: Probes empty, it looks like you have dumped "
|
||||
"them without calling vfc_put_probe"
|
||||
)
|
||||
warnings.append(execution_data)
|
||||
|
||||
# Once the CSV has been opened and validated, return its content
|
||||
results["value"] = results["value"].apply(lambda x: float.fromhex(x))
|
||||
results.rename(columns={"value": "values"}, inplace=True)
|
||||
|
||||
results["vfc_backend"] = backend
|
||||
|
||||
return results
|
||||
|
||||
|
||||
# Wrappers to sd.significant_digits (returns results in base 2)
|
||||
|
||||
def significant_digits(x):
|
||||
|
||||
# If the null hypothesis is rejected, call sigdigits with the General
|
||||
# formula:
|
||||
if x.pvalue < min_pvalue:
|
||||
# In a pandas DF, "values" actually refers to the array of columns, and
|
||||
# not the column named "values"
|
||||
distribution = x.values[3]
|
||||
distribution = distribution.reshape(len(distribution), 1)
|
||||
|
||||
# The distribution's empirical average will be used as the reference
|
||||
mu = np.array([x.mu])
|
||||
|
||||
s = sd.significant_digits(
|
||||
distribution,
|
||||
mu,
|
||||
precision=sd.Precision.Relative,
|
||||
method=sd.Method.General,
|
||||
|
||||
probability=0.9,
|
||||
confidence=0.95
|
||||
)
|
||||
|
||||
# s is returned inside a list
|
||||
return s[0]
|
||||
|
||||
# Else, manually compute sMCA (Stott-Parker formula)
|
||||
else:
|
||||
return -np.log2(np.absolute(x.sigma / x.mu))
|
||||
|
||||
|
||||
def significant_digits_lower_bound(x):
|
||||
# If the null hypothesis is rejected, no lower bound
|
||||
if x.pvalue < min_pvalue:
|
||||
return x.s2
|
||||
|
||||
# Else, the lower bound will be computed with p= .9 alpha-1=.95
|
||||
else:
|
||||
distribution = x.values[3]
|
||||
distribution = distribution.reshape(len(distribution), 1)
|
||||
|
||||
mu = np.array([x.mu])
|
||||
|
||||
s = sd.significant_digits(
|
||||
distribution,
|
||||
mu,
|
||||
precision=sd.Precision.Relative,
|
||||
method=sd.Method.CNH,
|
||||
|
||||
probability=0.9,
|
||||
confidence=0.95
|
||||
)
|
||||
|
||||
return s[0]
|
||||
|
||||
|
||||
##########################################################################
|
||||
|
||||
# Main functions
|
||||
|
||||
|
||||
# Open and read the tests config file
|
||||
def read_config():
|
||||
try:
|
||||
with open("vfc_tests_config.json", "r") as file:
|
||||
data = file.read()
|
||||
|
||||
except FileNotFoundError as e:
|
||||
e.strerror = "Error [vfc_ci]: This file is required to describe the tests "\
|
||||
"to run and generate a Verificarlo run file"
|
||||
raise e
|
||||
|
||||
return json.loads(data)
|
||||
|
||||
|
||||
# Set up metadata
|
||||
def generate_metadata(is_git_commit):
|
||||
|
||||
# Metadata and filename are initiated as if no commit was associated
|
||||
metadata = {
|
||||
"timestamp": calendar.timegm(time.gmtime()),
|
||||
"is_git_commit": is_git_commit,
|
||||
"hash": "",
|
||||
"author": "",
|
||||
"message": ""
|
||||
}
|
||||
|
||||
if is_git_commit:
|
||||
print("Fetching metadata from last commit...")
|
||||
from git import Repo
|
||||
|
||||
repo = Repo(".")
|
||||
head_commit = repo.head.commit
|
||||
|
||||
metadata["timestamp"] = head_commit.authored_date
|
||||
|
||||
metadata["hash"] = str(head_commit)[0:7]
|
||||
metadata["author"] = "%s <%s>" \
|
||||
% (str(head_commit.author), head_commit.author.email)
|
||||
metadata["message"] = head_commit.message.split("\n")[0]
|
||||
|
||||
return metadata
|
||||
|
||||
|
||||
# Execute tests and collect results in a Pandas dataframe (+ dataprocessing)
|
||||
def run_tests(config):
|
||||
|
||||
# Run the build command
|
||||
print("Info [vfc_ci]: Building tests...")
|
||||
os.system(config["make_command"])
|
||||
|
||||
# This is an array of Pandas dataframes for now
|
||||
data = []
|
||||
|
||||
# Create tmp folder to export results
|
||||
os.system("mkdir .vfcruns.tmp")
|
||||
n_files = 0
|
||||
|
||||
# This will contain all executables/repetition numbers from which we could
|
||||
# not get any data
|
||||
warnings = []
|
||||
|
||||
# Tests execution loop
|
||||
for executable in config["executables"]:
|
||||
print("Info [vfc_ci]: Running executable :",
|
||||
executable["executable"], "...")
|
||||
|
||||
parameters = ""
|
||||
if "parameters" in executable:
|
||||
parameters = executable["parameters"]
|
||||
|
||||
for backend in executable["vfc_backends"]:
|
||||
|
||||
export_backend = "VFC_BACKENDS=\"" + backend["name"] + "\" "
|
||||
command = "./" + executable["executable"] + " " + parameters
|
||||
|
||||
repetitions = 1
|
||||
if "repetitions" in backend:
|
||||
repetitions = backend["repetitions"]
|
||||
|
||||
# Run test repetitions and save results
|
||||
for i in range(repetitions):
|
||||
file = ".vfcruns.tmp/%s.csv" % str(n_files)
|
||||
export_output = "VFC_PROBES_OUTPUT=\"%s\" " % file
|
||||
os.system(export_output + export_backend + command)
|
||||
|
||||
# This will only be used if we need to append this exec to the
|
||||
# warnings list
|
||||
execution_data = {
|
||||
"executable": executable["executable"],
|
||||
"backend": backend["name"],
|
||||
"repetition": i + 1
|
||||
}
|
||||
|
||||
data.append(read_probes_csv(
|
||||
file,
|
||||
backend["name"],
|
||||
warnings,
|
||||
execution_data
|
||||
))
|
||||
|
||||
n_files = n_files + 1
|
||||
|
||||
# Clean CSV output files (by deleting the tmp folder)
|
||||
os.system("rm -rf .vfcruns.tmp")
|
||||
|
||||
# Combine all separate executions in one dataframe
|
||||
data = pd.concat(data, sort=False, ignore_index=True)
|
||||
data = data.groupby(["test", "vfc_backend", "variable"])\
|
||||
.values.apply(list).reset_index()
|
||||
|
||||
# Make sure we have some data to work on
|
||||
assert(len(data) != 0), "Error [vfc_ci]: No data have been generated " \
|
||||
"by your tests executions, aborting run without writing results file"
|
||||
|
||||
return data, warnings
|
||||
|
||||
# Data processing
|
||||
|
||||
|
||||
def data_processing(data):
|
||||
|
||||
data["values"] = data["values"].apply(lambda x: np.array(x).astype(float))
|
||||
|
||||
# Get empirical average, standard deviation and p-value
|
||||
data["mu"] = data["values"].apply(np.average)
|
||||
data["sigma"] = data["values"].apply(np.std)
|
||||
data["pvalue"] = data["values"].apply(
|
||||
lambda x: scipy.stats.shapiro(x).pvalue)
|
||||
|
||||
# Significant digits
|
||||
data["s2"] = data.apply(significant_digits, axis=1)
|
||||
data["s10"] = data["s2"].apply(lambda x: sd.change_base(x, 10))
|
||||
|
||||
# Lower bound of the confidence interval using the sigdigits module
|
||||
data["s2_lower_bound"] = data.apply(significant_digits_lower_bound, axis=1)
|
||||
data["s10_lower_bound"] = data["s2_lower_bound"].apply(
|
||||
lambda x: sd.change_base(x, 10))
|
||||
|
||||
# Compute moments of the distribution
|
||||
# (including a new distribution obtained by filtering outliers)
|
||||
data["values"] = data["values"].apply(np.sort)
|
||||
|
||||
data["mu"] = data["values"].apply(np.average)
|
||||
data["min"] = data["values"].apply(np.min)
|
||||
data["quantile25"] = data["values"].apply(np.quantile, args=(0.25,))
|
||||
data["quantile50"] = data["values"].apply(np.quantile, args=(0.50,))
|
||||
data["quantile75"] = data["values"].apply(np.quantile, args=(0.75,))
|
||||
data["max"] = data["values"].apply(np.max)
|
||||
|
||||
data["nsamples"] = data["values"].apply(len)
|
||||
|
||||
# Display all executions that resulted in a warning
|
||||
|
||||
|
||||
def show_warnings(warnings):
|
||||
if len(warnings) > 0:
|
||||
print(
|
||||
"Warning [vfc_ci]: Some of your runs could not generate any data "
|
||||
"(for instance because your code crashed) and resulted in "
|
||||
"warnings. Here is the complete list :"
|
||||
)
|
||||
|
||||
for i in range(0, len(warnings)):
|
||||
print("- Warning %s:" % i)
|
||||
|
||||
print(" Executable: %s" % warnings[i]["executable"])
|
||||
print(" Backend: %s" % warnings[i]["backend"])
|
||||
print(" Repetition: %s" % warnings[i]["repetition"])
|
||||
|
||||
|
||||
##########################################################################
|
||||
|
||||
# Entry point
|
||||
|
||||
def run(is_git_commit, export_raw_values, dry_run):
|
||||
|
||||
# Get config, metadata and data
|
||||
print("Info [vfc_ci]: Reading tests config file...")
|
||||
config = read_config()
|
||||
|
||||
print("Info [vfc_ci]: Generating run metadata...")
|
||||
metadata = generate_metadata(is_git_commit)
|
||||
|
||||
data, warnings = run_tests(config)
|
||||
show_warnings(warnings)
|
||||
|
||||
# Data processing
|
||||
print("Info [vfc_ci]: Processing data...")
|
||||
data_processing(data)
|
||||
|
||||
# Prepare data for export (by creating a proper index and linking run
|
||||
# timestamp)
|
||||
data = data.set_index(["test", "variable", "vfc_backend"]).sort_index()
|
||||
data["timestamp"] = metadata["timestamp"]
|
||||
|
||||
filename = metadata["hash"] if is_git_commit else str(
|
||||
metadata["timestamp"])
|
||||
|
||||
# Prepare metadata for export
|
||||
metadata = pd.DataFrame.from_dict([metadata])
|
||||
metadata = metadata.set_index("timestamp")
|
||||
|
||||
# NOTE : Exporting to HDF5 requires to install "tables" on the system
|
||||
|
||||
# Export raw data if needed
|
||||
if export_raw_values and not dry_run:
|
||||
data.to_hdf(filename + ".vfcraw.h5", key="data")
|
||||
metadata.to_hdf(filename + ".vfcraw.h5", key="metadata")
|
||||
|
||||
# Export data
|
||||
del data["values"]
|
||||
if not dry_run:
|
||||
data.to_hdf(filename + ".vfcrun.h5", key="data")
|
||||
metadata.to_hdf(filename + ".vfcrun.h5", key="metadata")
|
||||
|
||||
# Print termination messages
|
||||
print(
|
||||
"Info [vfc_ci]: The results have been successfully written to "
|
||||
"%s.vfcrun.h5."
|
||||
% filename
|
||||
)
|
||||
|
||||
if export_raw_values:
|
||||
print(
|
||||
"Info [vfc_ci]: A file containing the raw values has also been "
|
||||
"created : %s.vfcraw.h5."
|
||||
% filename
|
||||
)
|
||||
|
||||
if dry_run:
|
||||
print(
|
||||
"Info [vfc_ci]: The dry run flag was enabled, so no files were "
|
||||
"actually created."
|
||||
)
|
||||
@ -1,542 +0,0 @@
|
||||
# Manage the view comparing a variable over different runs
|
||||
|
||||
import time
|
||||
|
||||
import pandas as pd
|
||||
|
||||
from math import pi
|
||||
|
||||
from bokeh.plotting import figure, curdoc
|
||||
from bokeh.embed import components
|
||||
from bokeh.models import Select, ColumnDataSource, Panel, Tabs, HoverTool, \
|
||||
TextInput, CheckboxGroup, TapTool, CustomJS
|
||||
|
||||
import helper
|
||||
import plot
|
||||
|
||||
|
||||
##########################################################################
|
||||
|
||||
|
||||
class CompareRuns:
|
||||
|
||||
# Helper functions related to CompareRuns
|
||||
|
||||
# From an array of timestamps, returns the array of runs names (for the x
|
||||
# axis ticks), as well as the metadata (in a dict of arrays) associated to
|
||||
# this array (for the tooltips)
|
||||
def gen_x_series(self, timestamps):
|
||||
|
||||
# Initialize the objects to return
|
||||
x_series = []
|
||||
x_metadata = dict(
|
||||
date=[],
|
||||
is_git_commit=[],
|
||||
hash=[],
|
||||
author=[],
|
||||
message=[]
|
||||
)
|
||||
|
||||
# n == 0 means we want all runs, we also make sure not to go out of
|
||||
# bound if asked for more runs than we have
|
||||
n = self.current_n_runs
|
||||
if n == 0 or n > len(timestamps):
|
||||
n = len(timestamps)
|
||||
|
||||
for i in range(0, n):
|
||||
# Get metadata associated to this run
|
||||
row_metadata = helper.get_metadata(
|
||||
self.metadata, timestamps[-i - 1])
|
||||
date = time.ctime(timestamps[-i - 1])
|
||||
|
||||
# Fill the x series
|
||||
str = row_metadata["name"]
|
||||
x_series.insert(0, helper.get_metadata(
|
||||
self.metadata, timestamps[-i - 1])["name"])
|
||||
|
||||
# Fill the metadata lists
|
||||
x_metadata["date"].insert(0, date)
|
||||
x_metadata["is_git_commit"].insert(
|
||||
0, row_metadata["is_git_commit"])
|
||||
x_metadata["hash"].insert(0, row_metadata["hash"])
|
||||
x_metadata["author"].insert(0, row_metadata["author"])
|
||||
x_metadata["message"].insert(0, row_metadata["message"])
|
||||
|
||||
return x_series, x_metadata
|
||||
|
||||
# Plots update function
|
||||
|
||||
def update_plots(self):
|
||||
|
||||
# Select all data matching current test/var/backend
|
||||
|
||||
runs = self.data.loc[[self.widgets["select_test"].value],
|
||||
self.widgets["select_var"].value,
|
||||
self.widgets["select_backend"].value]
|
||||
|
||||
timestamps = runs["timestamp"]
|
||||
x_series, x_metadata = self.gen_x_series(timestamps.sort_values())
|
||||
|
||||
# Update source
|
||||
|
||||
main_dict = runs.to_dict("series")
|
||||
main_dict["x"] = x_series
|
||||
|
||||
# Add metadata (for tooltip)
|
||||
main_dict.update(x_metadata)
|
||||
|
||||
# Select the last n runs only
|
||||
n = self.current_n_runs
|
||||
main_dict = {key: value[-n:] for key, value in main_dict.items()}
|
||||
|
||||
# Generate ColumnDataSources for the 3 dotplots
|
||||
for stat in ["sigma", "s10", "s2"]:
|
||||
dict = {
|
||||
"%s_x" % stat: main_dict["x"],
|
||||
|
||||
"is_git_commit": main_dict["is_git_commit"],
|
||||
"date": main_dict["date"],
|
||||
"hash": main_dict["hash"],
|
||||
"author": main_dict["author"],
|
||||
"message": main_dict["message"],
|
||||
|
||||
stat: main_dict[stat],
|
||||
|
||||
"nsamples": main_dict["nsamples"],
|
||||
}
|
||||
|
||||
if stat == "s10" or stat == "s2":
|
||||
dict["%s_lower_bound" %
|
||||
stat] = main_dict["%s_lower_bound" %
|
||||
stat]
|
||||
|
||||
# Filter outliers if the box is checked
|
||||
if len(self.widgets["outliers_filtering_compare"].active) > 0:
|
||||
outliers = helper.detect_outliers(dict[stat])
|
||||
dict[stat] = helper.remove_outliers(dict[stat], outliers)
|
||||
dict["%s_x" % stat] = helper.remove_outliers(
|
||||
dict["%s_x" % stat], outliers)
|
||||
|
||||
# Assign ColumnDataSource
|
||||
self.sources["%s_source" % stat].data = dict
|
||||
|
||||
# Generate ColumnDataSource for the boxplot
|
||||
dict = {
|
||||
"is_git_commit": main_dict["is_git_commit"],
|
||||
"date": main_dict["date"],
|
||||
"hash": main_dict["hash"],
|
||||
"author": main_dict["author"],
|
||||
"message": main_dict["message"],
|
||||
|
||||
"x": main_dict["x"],
|
||||
"min": main_dict["min"],
|
||||
"quantile25": main_dict["quantile25"],
|
||||
"quantile50": main_dict["quantile50"],
|
||||
"quantile75": main_dict["quantile75"],
|
||||
"max": main_dict["max"],
|
||||
"mu": main_dict["mu"],
|
||||
"pvalue": main_dict["pvalue"],
|
||||
|
||||
"nsamples": main_dict["nsamples"]
|
||||
}
|
||||
|
||||
self.sources["boxplot_source"].data = dict
|
||||
|
||||
# Update x axis
|
||||
|
||||
helper.reset_x_range(
|
||||
self.plots["boxplot"],
|
||||
self.sources["boxplot_source"].data["x"]
|
||||
)
|
||||
helper.reset_x_range(
|
||||
self.plots["sigma_plot"],
|
||||
self.sources["sigma_source"].data["sigma_x"]
|
||||
)
|
||||
helper.reset_x_range(
|
||||
self.plots["s10_plot"],
|
||||
self.sources["s10_source"].data["s10_x"]
|
||||
)
|
||||
helper.reset_x_range(
|
||||
self.plots["s2_plot"],
|
||||
self.sources["s2_source"].data["s2_x"]
|
||||
)
|
||||
|
||||
# Widgets' callback functions
|
||||
|
||||
def update_test(self, attrname, old, new):
|
||||
|
||||
# If the value is updated by the CustomJS, self.widgets["select_var"].value
|
||||
# won't be updated, so we have to look for that case and assign it
|
||||
# manually
|
||||
|
||||
# "new" should be a list when updated by CustomJS
|
||||
if isinstance(new, list):
|
||||
# If filtering removed all options, we might have an empty list
|
||||
# (in this case, we just skip the callback and do nothing)
|
||||
if len(new) > 0:
|
||||
new = new[0]
|
||||
else:
|
||||
return
|
||||
|
||||
if new != self.widgets["select_test"].value:
|
||||
# The callback will be triggered again with the updated value
|
||||
self.widgets["select_test"].value = new
|
||||
return
|
||||
|
||||
# New list of available vars
|
||||
self.vars = self.data.loc[new]\
|
||||
.index.get_level_values("variable").drop_duplicates().tolist()
|
||||
self.widgets["select_var"].options = self.vars
|
||||
|
||||
# Reset var selection if old one is not available in new vars
|
||||
if self.widgets["select_var"].value not in self.vars:
|
||||
self.widgets["select_var"].value = self.vars[0]
|
||||
# The update_var callback will be triggered by the assignment
|
||||
|
||||
else:
|
||||
# Trigger the callback manually (since the plots need to be updated
|
||||
# anyway)
|
||||
self.update_var("", "", self.widgets["select_var"].value)
|
||||
|
||||
def update_var(self, attrname, old, new):
|
||||
|
||||
# If the value is updated by the CustomJS, self.widgets["select_var"].value
|
||||
# won't be updated, so we have to look for that case and assign it
|
||||
# manually
|
||||
|
||||
# new should be a list when updated by CustomJS
|
||||
if isinstance(new, list):
|
||||
new = new[0]
|
||||
|
||||
if new != self.widgets["select_var"].value:
|
||||
# The callback will be triggered again with the updated value
|
||||
self.widgets["select_var"].value = new
|
||||
return
|
||||
|
||||
# New list of available backends
|
||||
self.backends = self.data.loc[self.widgets["select_test"].value, self.widgets["select_var"].value]\
|
||||
.index.get_level_values("vfc_backend").drop_duplicates().tolist()
|
||||
self.widgets["select_backend"].options = self.backends
|
||||
|
||||
# Reset backend selection if old one is not available in new backends
|
||||
if self.widgets["select_backend"].value not in self.backends:
|
||||
self.widgets["select_backend"].value = self.backends[0]
|
||||
# The update_backend callback will be triggered by the assignment
|
||||
|
||||
else:
|
||||
# Trigger the callback manually (since the plots need to be updated
|
||||
# anyway)
|
||||
self.update_backend("", "", self.widgets["select_backend"].value)
|
||||
|
||||
def update_backend(self, attrname, old, new):
|
||||
|
||||
# Simply update plots, since no other data is affected
|
||||
self.update_plots()
|
||||
|
||||
def update_n_runs(self, attrname, old, new):
|
||||
# Simply update runs selection (value and string display)
|
||||
self.widgets["select_n_runs"].value = new
|
||||
self.current_n_runs = self.n_runs_dict[self.widgets["select_n_runs"].value]
|
||||
|
||||
self.update_plots()
|
||||
|
||||
def update_outliers_filtering(self, attrname, old, new):
|
||||
self.update_plots()
|
||||
|
||||
# Bokeh setup functions
|
||||
|
||||
def setup_plots(self):
|
||||
|
||||
tools = "pan, wheel_zoom, xwheel_zoom, ywheel_zoom, reset, save"
|
||||
|
||||
# Custom JS callback that will be used when tapping on a run
|
||||
# Only switches the view, a server callback is required to update plots
|
||||
# (defined inside template to avoid bloating server w/ too much JS code)
|
||||
js_tap_callback = "goToInspectRuns();"
|
||||
|
||||
# Box plot
|
||||
self.plots["boxplot"] = figure(
|
||||
name="boxplot", title="Variable distribution over runs",
|
||||
plot_width=900, plot_height=400, x_range=[""],
|
||||
tools=tools, sizing_mode="scale_width"
|
||||
)
|
||||
|
||||
box_tooltips = [
|
||||
("Git commit", "@is_git_commit"),
|
||||
("Date", "@date"),
|
||||
("Hash", "@hash"),
|
||||
("Author", "@author"),
|
||||
("Message", "@message"),
|
||||
("Min", "@min{%0.18e}"),
|
||||
("Max", "@max{%0.18e}"),
|
||||
("1st quartile", "@quantile25{%0.18e}"),
|
||||
("Median", "@quantile50{%0.18e}"),
|
||||
("3rd quartile", "@quantile75{%0.18e}"),
|
||||
("μ", "@mu{%0.18e}"),
|
||||
("p-value", "@pvalue"),
|
||||
("Number of samples", "@nsamples")
|
||||
]
|
||||
box_tooltips_formatters = {
|
||||
"@min": "printf",
|
||||
"@max": "printf",
|
||||
"@quantile25": "printf",
|
||||
"@quantile50": "printf",
|
||||
"@quantile75": "printf",
|
||||
"@mu": "printf"
|
||||
}
|
||||
|
||||
plot.fill_boxplot(
|
||||
self.plots["boxplot"], self.sources["boxplot_source"],
|
||||
tooltips=box_tooltips,
|
||||
tooltips_formatters=box_tooltips_formatters,
|
||||
js_tap_callback=js_tap_callback,
|
||||
server_tap_callback=self.inspect_run_callback_boxplot,
|
||||
)
|
||||
self.doc.add_root(self.plots["boxplot"])
|
||||
|
||||
# Sigma plot (bar plot)
|
||||
self.plots["sigma_plot"] = figure(
|
||||
name="sigma_plot", title="Standard deviation σ over runs",
|
||||
plot_width=900, plot_height=400, x_range=[""],
|
||||
tools=tools, sizing_mode="scale_width"
|
||||
)
|
||||
|
||||
sigma_tooltips = [
|
||||
("Git commit", "@is_git_commit"),
|
||||
("Date", "@date"),
|
||||
("Hash", "@hash"),
|
||||
("Author", "@author"),
|
||||
("Message", "@message"),
|
||||
("σ", "@sigma"),
|
||||
("Number of samples", "@nsamples")
|
||||
]
|
||||
|
||||
plot.fill_dotplot(
|
||||
self.plots["sigma_plot"], self.sources["sigma_source"], "sigma",
|
||||
tooltips=sigma_tooltips,
|
||||
js_tap_callback=js_tap_callback,
|
||||
server_tap_callback=self.inspect_run_callback_sigma,
|
||||
lines=True
|
||||
)
|
||||
self.doc.add_root(self.plots["sigma_plot"])
|
||||
|
||||
# s plot (bar plot with 2 tabs)
|
||||
self.plots["s10_plot"] = figure(
|
||||
name="s10_plot", title="Significant digits s over runs",
|
||||
plot_width=900, plot_height=400, x_range=[""],
|
||||
tools=tools, sizing_mode="scale_width"
|
||||
)
|
||||
|
||||
s10_tooltips = [
|
||||
("Git commit", "@is_git_commit"),
|
||||
("Date", "@date"),
|
||||
("Hash", "@hash"),
|
||||
("Author", "@author"),
|
||||
("Message", "@message"),
|
||||
("s", "@s10"),
|
||||
("s lower bound", "@s10_lower_bound"),
|
||||
("Number of samples", "@nsamples")
|
||||
]
|
||||
|
||||
plot.fill_dotplot(
|
||||
self.plots["s10_plot"], self.sources["s10_source"], "s10",
|
||||
tooltips=s10_tooltips,
|
||||
js_tap_callback=js_tap_callback,
|
||||
server_tap_callback=self.inspect_run_callback_s10,
|
||||
lines=True,
|
||||
lower_bound=True
|
||||
)
|
||||
s10_tab = Panel(child=self.plots["s10_plot"], title="Base 10")
|
||||
|
||||
self.plots["s2_plot"] = figure(
|
||||
name="s2_plot", title="Significant digits s over runs",
|
||||
plot_width=900, plot_height=400, x_range=[""],
|
||||
tools=tools, sizing_mode="scale_width"
|
||||
)
|
||||
|
||||
s2_tooltips = [
|
||||
("Git commit", "@is_git_commit"),
|
||||
("Date", "@date"),
|
||||
("Hash", "@hash"),
|
||||
("Author", "@author"),
|
||||
("Message", "@message"),
|
||||
("s", "@s2"),
|
||||
("s lower bound", "@s2_lower_bound"),
|
||||
("Number of samples", "@nsamples")
|
||||
]
|
||||
|
||||
plot.fill_dotplot(
|
||||
self.plots["s2_plot"], self.sources["s2_source"], "s2",
|
||||
tooltips=s2_tooltips,
|
||||
js_tap_callback=js_tap_callback,
|
||||
server_tap_callback=self.inspect_run_callback_s2,
|
||||
lines=True,
|
||||
lower_bound=True
|
||||
)
|
||||
s2_tab = Panel(child=self.plots["s2_plot"], title="Base 2")
|
||||
|
||||
s_tabs = Tabs(
|
||||
name="s_tabs",
|
||||
tabs=[s10_tab, s2_tab],
|
||||
tabs_location="below"
|
||||
)
|
||||
|
||||
self.doc.add_root(s_tabs)
|
||||
|
||||
def setup_widgets(self):
|
||||
|
||||
# Initial selections
|
||||
|
||||
# Test/var/backend combination (we select all first elements at init)
|
||||
self.tests = self.data\
|
||||
.index.get_level_values("test").drop_duplicates().tolist()
|
||||
|
||||
self.vars = self.data.loc[self.tests[0]]\
|
||||
.index.get_level_values("variable").drop_duplicates().tolist()
|
||||
|
||||
self.backends = self.data.loc[self.tests[0], self.vars[0]]\
|
||||
.index.get_level_values("vfc_backend").drop_duplicates().tolist()
|
||||
|
||||
# Custom JS callback that will be used client side to filter selections
|
||||
filter_callback_js = """
|
||||
selector.options = options.filter(e => e.includes(cb_obj.value));
|
||||
"""
|
||||
|
||||
# Test selector widget
|
||||
|
||||
# Number of runs to display
|
||||
# The dict structure allows us to get int value from the display string
|
||||
# in O(1)
|
||||
self.n_runs_dict = {
|
||||
"Last 3 runs": 3,
|
||||
"Last 5 runs": 5,
|
||||
"Last 10 runs": 10,
|
||||
"All runs": 0
|
||||
}
|
||||
|
||||
# Contains all options strings
|
||||
n_runs_display = list(self.n_runs_dict.keys())
|
||||
|
||||
# Will be used when updating plots (contains actual number to diplay)
|
||||
self.current_n_runs = self.n_runs_dict[n_runs_display[1]]
|
||||
|
||||
# Selector widget
|
||||
self.widgets["select_test"] = Select(
|
||||
name="select_test", title="Test :",
|
||||
value=self.tests[0], options=self.tests
|
||||
)
|
||||
self.doc.add_root(self.widgets["select_test"])
|
||||
self.widgets["select_test"].on_change("value", self.update_test)
|
||||
self.widgets["select_test"].on_change("options", self.update_test)
|
||||
|
||||
# Filter widget
|
||||
self.widgets["test_filter"] = TextInput(
|
||||
name="test_filter", title="Tests filter:"
|
||||
)
|
||||
self.widgets["test_filter"].js_on_change(
|
||||
"value",
|
||||
CustomJS(
|
||||
args=dict(
|
||||
options=self.tests,
|
||||
selector=self.widgets["select_test"]),
|
||||
code=filter_callback_js))
|
||||
self.doc.add_root(self.widgets["test_filter"])
|
||||
|
||||
# Number of runs to display
|
||||
|
||||
self.widgets["select_n_runs"] = Select(
|
||||
name="select_n_runs", title="Display :",
|
||||
value=n_runs_display[1], options=n_runs_display
|
||||
)
|
||||
self.doc.add_root(self.widgets["select_n_runs"])
|
||||
self.widgets["select_n_runs"].on_change("value", self.update_n_runs)
|
||||
|
||||
# Variable selector widget
|
||||
|
||||
self.widgets["select_var"] = Select(
|
||||
name="select_var", title="Variable :",
|
||||
value=self.vars[0], options=self.vars
|
||||
)
|
||||
self.doc.add_root(self.widgets["select_var"])
|
||||
self.widgets["select_var"].on_change("value", self.update_var)
|
||||
self.widgets["select_var"].on_change("options", self.update_var)
|
||||
|
||||
# Backend selector widget
|
||||
|
||||
self.widgets["select_backend"] = Select(
|
||||
name="select_backend", title="Verificarlo backend :",
|
||||
value=self.backends[0], options=self.backends
|
||||
)
|
||||
self.doc.add_root(self.widgets["select_backend"])
|
||||
self.widgets["select_backend"].on_change("value", self.update_backend)
|
||||
|
||||
# Outliers filtering checkbox
|
||||
|
||||
self.widgets["outliers_filtering_compare"] = CheckboxGroup(
|
||||
name="outliers_filtering_compare",
|
||||
labels=["Filter outliers"], active=[]
|
||||
)
|
||||
self.doc.add_root(self.widgets["outliers_filtering_compare"])
|
||||
self.widgets["outliers_filtering_compare"]\
|
||||
.on_change("active", self.update_outliers_filtering)
|
||||
|
||||
# Communication methods
|
||||
# (to send/receive messages to/from master)
|
||||
|
||||
# Callback to change view of Inspect runs when data is selected
|
||||
|
||||
def inspect_run_callback(self, new, source_name, x_name):
|
||||
|
||||
# In case we just unselected everything, then do nothing
|
||||
if new == []:
|
||||
return
|
||||
|
||||
index = new[-1]
|
||||
run_name = self.sources[source_name].data[x_name][index]
|
||||
|
||||
self.master.go_to_inspect(run_name)
|
||||
|
||||
# Wrappers for each plot (since new is the index of the clicked element,
|
||||
# it is dependent of the plot because we could have filtered some outliers)
|
||||
# There doesn't seem to be an easy way to add custom parameters to a
|
||||
# Bokeh callback, so using wrappers seems to be the best solution for now
|
||||
|
||||
def inspect_run_callback_boxplot(self, attr, old, new):
|
||||
self.inspect_run_callback(new, "boxplot_source", "x")
|
||||
|
||||
def inspect_run_callback_sigma(self, attr, old, new):
|
||||
self.inspect_run_callback(new, "sigma_source", "sigma_x")
|
||||
|
||||
def inspect_run_callback_s2(self, attr, old, new):
|
||||
self.inspect_run_callback(new, "s2_source", "s2_x")
|
||||
|
||||
def inspect_run_callback_s10(self, attr, old, new):
|
||||
self.inspect_run_callback(new, "s10_source", "s10_x")
|
||||
|
||||
# Constructor
|
||||
|
||||
def __init__(self, master, doc, data, metadata):
|
||||
|
||||
self.master = master
|
||||
|
||||
self.doc = doc
|
||||
self.data = data
|
||||
self.metadata = metadata
|
||||
|
||||
self.sources = {
|
||||
"boxplot_source": ColumnDataSource(data={}),
|
||||
"sigma_source": ColumnDataSource(data={}),
|
||||
"s10_source": ColumnDataSource(data={}),
|
||||
"s2_source": ColumnDataSource(data={})
|
||||
}
|
||||
|
||||
self.plots = {}
|
||||
self.widgets = {}
|
||||
|
||||
# Setup Bokeh objects
|
||||
self.setup_plots()
|
||||
self.setup_widgets()
|
||||
|
||||
# At this point, everything should have been initialized, so we can
|
||||
# show the plots for the first time
|
||||
self.update_plots()
|
||||
@ -1,170 +0,0 @@
|
||||
# General helper functions for both compare_runs and compare_variables
|
||||
|
||||
import calendar
|
||||
import time
|
||||
from itertools import compress
|
||||
|
||||
import numpy as np
|
||||
|
||||
# Magic numbers
|
||||
max_ticks = 15
|
||||
max_zscore = 3
|
||||
|
||||
##########################################################################
|
||||
|
||||
|
||||
# From a timestamp, return the associated metadata as a Pandas serie
|
||||
def get_metadata(metadata, timestamp):
|
||||
return metadata.loc[timestamp]
|
||||
|
||||
|
||||
# Convert a metadata Pandas series to a JS readable dict
|
||||
def metadata_to_dict(metadata):
|
||||
dict = metadata.to_dict()
|
||||
|
||||
# JS doesn't accept True for booleans, and Python doesn't accept true
|
||||
# (because of the caps) => using an integer is a portable solution
|
||||
dict["is_git_commit"] = 1 if dict["is_git_commit"] else 0
|
||||
|
||||
dict["date"] = time.ctime(metadata.name)
|
||||
|
||||
return dict
|
||||
|
||||
|
||||
# Return a string that indicates the elapsed time since the run, used as the
|
||||
# x-axis tick in "Compare runs" or when selecting run in "Inspect run"
|
||||
def get_run_name(timestamp, hash):
|
||||
|
||||
gmt = time.gmtime()
|
||||
now = calendar.timegm(gmt)
|
||||
diff = now - timestamp
|
||||
|
||||
# Special case : < 1 minute (return string directly)
|
||||
if diff < 60:
|
||||
str = "Less than a minute ago"
|
||||
|
||||
if hash != "":
|
||||
str = str + " (%s)" % hash
|
||||
|
||||
if str == get_run_name.previous:
|
||||
get_run_name.counter = get_run_name.counter + 1
|
||||
str = "%s (%s)" % (str, get_run_name.counter)
|
||||
else:
|
||||
get_run_name.counter = 0
|
||||
get_run_name.previous = str
|
||||
|
||||
return str
|
||||
|
||||
# < 1 hour
|
||||
if diff < 3600:
|
||||
n = int(diff / 60)
|
||||
str = "%s minute%s ago"
|
||||
# < 1 day
|
||||
elif diff < 86400:
|
||||
n = int(diff / 3600)
|
||||
str = "%s hour%s ago"
|
||||
# < 1 week
|
||||
elif diff < 604800:
|
||||
n = int(diff / 86400)
|
||||
str = "%s day%s ago"
|
||||
# < 1 month
|
||||
elif diff < 2592000:
|
||||
n = int(diff / 604800)
|
||||
str = "%s week%s ago"
|
||||
# > 1 month
|
||||
else:
|
||||
n = diff / 2592000
|
||||
str = "%s month%s ago"
|
||||
|
||||
plural = ""
|
||||
if n != 1:
|
||||
plural = "s"
|
||||
|
||||
str = str % (n, plural)
|
||||
|
||||
# We might want to add the git hash
|
||||
if hash != "":
|
||||
str = str + " (%s)" % hash
|
||||
|
||||
# Finally, check for duplicate with previously generated string
|
||||
if str == get_run_name.previous:
|
||||
# Increment the duplicate counter and add it to str
|
||||
get_run_name.counter = get_run_name.counter + 1
|
||||
str = "%s (%s)" % (str, get_run_name.counter)
|
||||
|
||||
else:
|
||||
# No duplicate, reset both previously generated str and duplicate
|
||||
# counter
|
||||
get_run_name.counter = 0
|
||||
get_run_name.previous = str
|
||||
|
||||
return str
|
||||
|
||||
|
||||
# These external variables will store data about the last generated string to
|
||||
# avoid duplicates (assuming the runs are sorted by time)
|
||||
get_run_name.counter = 0
|
||||
get_run_name.previous = ""
|
||||
|
||||
|
||||
def reset_run_strings():
|
||||
get_run_name.counter = 0
|
||||
get_run_name.previous = ""
|
||||
|
||||
|
||||
# Update all the x-ranges from a dict of plots
|
||||
def reset_x_range(plot, x_range):
|
||||
plot.x_range.factors = x_range
|
||||
|
||||
if len(x_range) < max_ticks:
|
||||
plot.xaxis.major_tick_line_color = "#000000"
|
||||
plot.xaxis.minor_tick_line_color = "#000000"
|
||||
|
||||
plot.xaxis.major_label_text_font_size = "8pt"
|
||||
|
||||
else:
|
||||
plot.xaxis.major_tick_line_color = None
|
||||
plot.xaxis.minor_tick_line_color = None
|
||||
|
||||
plot.xaxis.major_label_text_font_size = "0pt"
|
||||
|
||||
|
||||
# Return an array of booleans that indicate which elements are outliers
|
||||
# (True means element is not an outlier and must be kept)
|
||||
def detect_outliers(array, max_zscore=max_zscore):
|
||||
if len(array) <= 2:
|
||||
return [True] * len(array)
|
||||
|
||||
median = np.median(array)
|
||||
std = np.std(array)
|
||||
if std == 0:
|
||||
return array
|
||||
distance = abs(array - median)
|
||||
# Array of booleans with elements to be filtered
|
||||
outliers_array = distance < max_zscore * std
|
||||
|
||||
return outliers_array
|
||||
|
||||
|
||||
def remove_outliers(array, outliers):
|
||||
return list(compress(array, outliers))
|
||||
|
||||
|
||||
def remove_boxplot_outliers(dict, outliers, prefix):
|
||||
outliers = detect_outliers(dict["%s_max" % prefix])
|
||||
|
||||
dict["%s_x" % prefix] = remove_outliers(dict["%s_x" % prefix], outliers)
|
||||
|
||||
dict["%s_min" % prefix] = remove_outliers(
|
||||
dict["%s_min" % prefix], outliers)
|
||||
dict["%s_quantile25" % prefix] = remove_outliers(
|
||||
dict["%s_quantile25" % prefix], outliers)
|
||||
dict["%s_quantile50" % prefix] = remove_outliers(
|
||||
dict["%s_quantile50" % prefix], outliers)
|
||||
dict["%s_quantile75" % prefix] = remove_outliers(
|
||||
dict["%s_quantile75" % prefix], outliers)
|
||||
dict["%s_max" % prefix] = remove_outliers(
|
||||
dict["%s_max" % prefix], outliers)
|
||||
dict["%s_mu" % prefix] = remove_outliers(dict["%s_mu" % prefix], outliers)
|
||||
|
||||
dict["nsamples"] = remove_outliers(dict["nsamples"], outliers)
|
||||
@ -1,570 +0,0 @@
|
||||
# Manage the view comparing the variables of a run
|
||||
|
||||
from math import pi
|
||||
from functools import partial
|
||||
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
|
||||
from bokeh.plotting import figure, curdoc
|
||||
from bokeh.embed import components
|
||||
from bokeh.models import Select, ColumnDataSource, Panel, Tabs, HoverTool,\
|
||||
RadioButtonGroup, CheckboxGroup, CustomJS
|
||||
|
||||
import helper
|
||||
import plot
|
||||
|
||||
|
||||
##########################################################################
|
||||
|
||||
|
||||
class InspectRuns:
|
||||
|
||||
# Helper functions related to InspectRun
|
||||
|
||||
# Returns a dictionary mapping user-readable strings to all run timestamps
|
||||
def gen_runs_selection(self):
|
||||
|
||||
runs_dict = {}
|
||||
|
||||
# Iterate over timestamp rows (runs) and fill dict
|
||||
for row in self.metadata.iloc:
|
||||
# The syntax used by pandas makes this part a bit tricky :
|
||||
# row.name is the index of metadata (so it refers to the
|
||||
# timestamp), whereas row["name"] is the column called "name"
|
||||
# (which is the display string used for the run)
|
||||
|
||||
# runs_dict[run's name] = run's timestamp
|
||||
runs_dict[row["name"]] = row.name
|
||||
|
||||
return runs_dict
|
||||
|
||||
def gen_boxplot_tooltips(self, prefix):
|
||||
return [
|
||||
("Name", "@%s_x" % prefix),
|
||||
("Min", "@" + prefix + "_min{%0.18e}"),
|
||||
("Max", "@" + prefix + "_max{%0.18e}"),
|
||||
("1st quartile", "@" + prefix + "_quantile25{%0.18e}"),
|
||||
("Median", "@" + prefix + "_quantile50{%0.18e}"),
|
||||
("3rd quartile", "@" + prefix + "_quantile75{%0.18e}"),
|
||||
("μ", "@" + prefix + "_mu{%0.18e}"),
|
||||
("Number of samples (tests)", "@nsamples")
|
||||
]
|
||||
|
||||
def gen_boxplot_tooltips_formatters(self, prefix):
|
||||
return {
|
||||
"@%s_min" % prefix: "printf",
|
||||
"@%s_max" % prefix: "printf",
|
||||
"@%s_quantile25" % prefix: "printf",
|
||||
"@%s_quantile50" % prefix: "printf",
|
||||
"@%s_quantile75" % prefix: "printf",
|
||||
"@%s_mu" % prefix: "printf"
|
||||
}
|
||||
|
||||
# Data processing helper
|
||||
# (computes new distributions for sigma, s2, s10)
|
||||
|
||||
def data_processing(self, dataframe):
|
||||
|
||||
# Compute aggragated mu
|
||||
dataframe["mu"] = np.vectorize(
|
||||
np.average)(
|
||||
dataframe["mu"],
|
||||
weights=dataframe["nsamples"])
|
||||
|
||||
# nsamples is the number of aggregated elements (as well as the number
|
||||
# of samples for our new sigma and s distributions)
|
||||
dataframe["nsamples"] = dataframe["nsamples"].apply(lambda x: len(x))
|
||||
|
||||
dataframe["mu_x"] = dataframe.index
|
||||
# Make sure that strings don't excede a certain length
|
||||
dataframe["mu_x"] = dataframe["mu_x"].apply(
|
||||
lambda x: x[:17] + "[...]" + x[-17:] if len(x) > 39 else x
|
||||
)
|
||||
|
||||
# Get quantiles and mu for sigma, s10, s2
|
||||
for prefix in ["sigma", "s10", "s2"]:
|
||||
|
||||
dataframe["%s_x" % prefix] = dataframe["mu_x"]
|
||||
|
||||
dataframe[prefix] = dataframe[prefix].apply(np.sort)
|
||||
|
||||
dataframe["%s_min" % prefix] = dataframe[prefix].apply(np.min)
|
||||
dataframe["%s_quantile25" % prefix] = dataframe[prefix].apply(
|
||||
np.quantile, args=(0.25,))
|
||||
dataframe["%s_quantile50" % prefix] = dataframe[prefix].apply(
|
||||
np.quantile, args=(0.50,))
|
||||
dataframe["%s_quantile75" % prefix] = dataframe[prefix].apply(
|
||||
np.quantile, args=(0.75,))
|
||||
dataframe["%s_max" % prefix] = dataframe[prefix].apply(np.max)
|
||||
dataframe["%s_mu" % prefix] = dataframe[prefix].apply(np.average)
|
||||
del dataframe[prefix]
|
||||
|
||||
return dataframe
|
||||
|
||||
# Plots update function
|
||||
|
||||
def update_plots(self):
|
||||
|
||||
groupby_display = self.widgets["groupby_radio"].labels[
|
||||
self.widgets["groupby_radio"].active
|
||||
]
|
||||
groupby = self.factors_dict[groupby_display]
|
||||
|
||||
filterby_display = self.widgets["filterby_radio"].labels[
|
||||
self.widgets["filterby_radio"].active
|
||||
]
|
||||
filterby = self.factors_dict[filterby_display]
|
||||
|
||||
# Groupby and aggregate lines belonging to the same group in lists
|
||||
|
||||
groups = self.run_data[
|
||||
self.run_data.index.isin(
|
||||
[self.widgets["select_filter"].value],
|
||||
level=filterby
|
||||
)
|
||||
].groupby(groupby)
|
||||
|
||||
groups = groups.agg({
|
||||
"sigma": lambda x: x.tolist(),
|
||||
"s10": lambda x: x.tolist(),
|
||||
"s2": lambda x: x.tolist(),
|
||||
|
||||
"mu": lambda x: x.tolist(),
|
||||
|
||||
# Used for mu weighted average first, then will be replaced
|
||||
"nsamples": lambda x: x.tolist()
|
||||
})
|
||||
|
||||
# Compute the new distributions, ...
|
||||
groups = self.data_processing(groups).to_dict("list")
|
||||
|
||||
# Update source
|
||||
|
||||
# Assign each ColumnDataSource, starting with the boxplots
|
||||
for prefix in ["sigma", "s10", "s2"]:
|
||||
|
||||
dict = {
|
||||
"%s_x" % prefix: groups["%s_x" % prefix],
|
||||
"%s_min" % prefix: groups["%s_min" % prefix],
|
||||
"%s_quantile25" % prefix: groups["%s_quantile25" % prefix],
|
||||
"%s_quantile50" % prefix: groups["%s_quantile50" % prefix],
|
||||
"%s_quantile75" % prefix: groups["%s_quantile75" % prefix],
|
||||
"%s_max" % prefix: groups["%s_max" % prefix],
|
||||
"%s_mu" % prefix: groups["%s_mu" % prefix],
|
||||
|
||||
"nsamples": groups["nsamples"]
|
||||
}
|
||||
|
||||
# Filter outliers if the box is checked
|
||||
if len(self.widgets["outliers_filtering_inspect"].active) > 0:
|
||||
|
||||
# Boxplots will be filtered by max then min
|
||||
top_outliers = helper.detect_outliers(dict["%s_max" % prefix])
|
||||
helper.remove_boxplot_outliers(dict, top_outliers, prefix)
|
||||
|
||||
bottom_outliers = helper.detect_outliers(
|
||||
dict["%s_min" % prefix])
|
||||
helper.remove_boxplot_outliers(dict, bottom_outliers, prefix)
|
||||
|
||||
self.sources["%s_source" % prefix].data = dict
|
||||
|
||||
# Finish with the mu plot
|
||||
dict = {
|
||||
"mu_x": groups["mu_x"],
|
||||
"mu": groups["mu"],
|
||||
|
||||
"nsamples": groups["nsamples"]
|
||||
}
|
||||
|
||||
self.sources["mu_source"].data = dict
|
||||
|
||||
# Filter outliers if the box is checked
|
||||
if len(self.widgets["outliers_filtering_inspect"].active) > 0:
|
||||
mu_outliers = helper.detect_outliers(groups["mu"])
|
||||
groups["mu"] = helper.remove_outliers(groups["mu"], mu_outliers)
|
||||
groups["mu_x"] = helper.remove_outliers(
|
||||
groups["mu_x"], mu_outliers)
|
||||
|
||||
# Update plots axis/titles
|
||||
|
||||
# Get display string of the last (unselected) factor
|
||||
factors_dict = self.factors_dict.copy()
|
||||
del factors_dict[groupby_display]
|
||||
del factors_dict[filterby_display]
|
||||
for_all = list(factors_dict.keys())[0]
|
||||
|
||||
# Update all display strings for plot title (remove caps, plural)
|
||||
groupby_display = groupby_display.lower()
|
||||
filterby_display = filterby_display.lower()[:-1]
|
||||
for_all = for_all.lower()
|
||||
|
||||
self.plots["mu_inspect"].title.text = \
|
||||
"Empirical average μ of %s (groupped by %s, for all %s)" \
|
||||
% (filterby_display, groupby_display, for_all)
|
||||
|
||||
self.plots["sigma_inspect"].title.text = \
|
||||
"Standard deviation σ of %s (groupped by %s, for all %s)" \
|
||||
% (filterby_display, groupby_display, for_all)
|
||||
|
||||
self.plots["s10_inspect"].title.text = \
|
||||
"Significant digits s of %s (groupped by %s, for all %s)" \
|
||||
% (filterby_display, groupby_display, for_all)
|
||||
|
||||
self.plots["s2_inspect"].title.text = \
|
||||
"Significant digits s of %s (groupped by %s, for all %s)" \
|
||||
% (filterby_display, groupby_display, for_all)
|
||||
|
||||
helper.reset_x_range(self.plots["mu_inspect"], groups["mu_x"])
|
||||
helper.reset_x_range(self.plots["sigma_inspect"], groups["sigma_x"])
|
||||
helper.reset_x_range(self.plots["s10_inspect"], groups["s10_x"])
|
||||
helper.reset_x_range(self.plots["s2_inspect"], groups["s2_x"])
|
||||
|
||||
# Widets' callback functions
|
||||
|
||||
# Run selector callback
|
||||
|
||||
def update_run(self, attrname, old, new):
|
||||
|
||||
filterby = self.widgets["filterby_radio"].labels[
|
||||
self.widgets["filterby_radio"].active
|
||||
]
|
||||
filterby = self.factors_dict[filterby]
|
||||
|
||||
# Update run selection (by using dict mapping)
|
||||
self.current_run = self.runs_dict[new]
|
||||
|
||||
# Update run data
|
||||
self.run_data = self.data[self.data["timestamp"] == self.current_run]
|
||||
|
||||
# Save old selected option
|
||||
old_value = self.widgets["select_filter"].value
|
||||
|
||||
# Update filter options
|
||||
options = self.run_data.index\
|
||||
.get_level_values(filterby).drop_duplicates().tolist()
|
||||
self.widgets["select_filter"].options = options
|
||||
|
||||
if old_value not in self.widgets["select_filter"].options:
|
||||
self.widgets["select_filter"].value = options[0]
|
||||
# The update_var callback will be triggered by the assignment
|
||||
|
||||
else:
|
||||
# Trigger the callback manually (since the plots need to be updated
|
||||
# anyway)
|
||||
self.update_filter("", "", old_value)
|
||||
|
||||
# "Group by" radio
|
||||
|
||||
def update_groupby(self, attrname, old, new):
|
||||
|
||||
# Update "Filter by" radio list
|
||||
filterby_list = list(self.factors_dict.keys())
|
||||
del filterby_list[self.widgets["groupby_radio"].active]
|
||||
self.widgets["filterby_radio"].labels = filterby_list
|
||||
|
||||
filterby = self.widgets["filterby_radio"].labels[
|
||||
self.widgets["filterby_radio"].active
|
||||
]
|
||||
filterby = self.factors_dict[filterby]
|
||||
|
||||
# Save old selected option
|
||||
old_value = self.widgets["select_filter"].value
|
||||
|
||||
# Update filter options
|
||||
options = self.run_data.index\
|
||||
.get_level_values(filterby).drop_duplicates().tolist()
|
||||
self.widgets["select_filter"].options = options
|
||||
|
||||
if old_value not in self.widgets["select_filter"].options:
|
||||
self.widgets["select_filter"].value = options[0]
|
||||
# The update_var callback will be triggered by the assignment
|
||||
|
||||
else:
|
||||
# Trigger the callback manually (since the plots need to be updated
|
||||
# anyway)
|
||||
self.update_filter("", "", old_value)
|
||||
|
||||
# "Filter by" radio
|
||||
|
||||
def update_filterby(self, attrname, old, new):
|
||||
|
||||
filterby = self.widgets["filterby_radio"].labels[
|
||||
self.widgets["filterby_radio"].active
|
||||
]
|
||||
filterby = self.factors_dict[filterby]
|
||||
|
||||
# Save old selected option
|
||||
old_value = self.widgets["select_filter"].value
|
||||
|
||||
# Update filter selector options
|
||||
options = self.run_data.index\
|
||||
.get_level_values(filterby).drop_duplicates().tolist()
|
||||
self.widgets["select_filter"].options = options
|
||||
|
||||
if old_value not in self.widgets["select_filter"].options:
|
||||
self.widgets["select_filter"].value = options[0]
|
||||
# The update_var callback will be triggered by the assignment
|
||||
|
||||
else:
|
||||
# Trigger the callback manually (since the plots need to be updated
|
||||
# anyway)
|
||||
self.update_filter("", "", old_value)
|
||||
|
||||
# Filter selector callback
|
||||
|
||||
def update_filter(self, attrname, old, new):
|
||||
self.update_plots()
|
||||
|
||||
# Filter outliers checkbox callback
|
||||
|
||||
def update_outliers_filtering(self, attrname, old, new):
|
||||
# The status (checked/unchecked) of the checkbox is also verified inside
|
||||
# self.update_plots(), so calling this function is enough
|
||||
self.update_plots()
|
||||
|
||||
# Bokeh setup functions
|
||||
# (for both variable and backend selection at once)
|
||||
|
||||
def setup_plots(self):
|
||||
|
||||
tools = "pan, wheel_zoom, xwheel_zoom, ywheel_zoom, reset, save"
|
||||
|
||||
# Tooltips and formatters
|
||||
|
||||
dotplot_tooltips = [
|
||||
("Name", "@mu_x"),
|
||||
("μ", "@mu{%0.18e}"),
|
||||
("Number of samples (tests)", "@nsamples")
|
||||
]
|
||||
dotplot_formatters = {
|
||||
"@mu": "printf"
|
||||
}
|
||||
|
||||
sigma_boxplot_tooltips = self.gen_boxplot_tooltips("sigma")
|
||||
sigma_boxplot_tooltips_formatters = self.gen_boxplot_tooltips_formatters(
|
||||
"sigma")
|
||||
|
||||
s10_boxplot_tooltips = self.gen_boxplot_tooltips("s10")
|
||||
s10_boxplot_tooltips_formatters = self.gen_boxplot_tooltips_formatters(
|
||||
"s10")
|
||||
|
||||
s2_boxplot_tooltips = self.gen_boxplot_tooltips("s2")
|
||||
s2_boxplot_tooltips_formatters = self.gen_boxplot_tooltips_formatters(
|
||||
"s2")
|
||||
|
||||
# Plots
|
||||
|
||||
# Mu plot
|
||||
self.plots["mu_inspect"] = figure(
|
||||
name="mu_inspect",
|
||||
title="",
|
||||
plot_width=900, plot_height=400, x_range=[""],
|
||||
tools=tools, sizing_mode="scale_width"
|
||||
)
|
||||
plot.fill_dotplot(
|
||||
self.plots["mu_inspect"], self.sources["mu_source"], "mu",
|
||||
tooltips=dotplot_tooltips,
|
||||
tooltips_formatters=dotplot_formatters
|
||||
)
|
||||
self.doc.add_root(self.plots["mu_inspect"])
|
||||
|
||||
# Sigma plot
|
||||
self.plots["sigma_inspect"] = figure(
|
||||
name="sigma_inspect",
|
||||
title="",
|
||||
plot_width=900, plot_height=400, x_range=[""],
|
||||
tools=tools, sizing_mode="scale_width"
|
||||
)
|
||||
plot.fill_boxplot(
|
||||
self.plots["sigma_inspect"],
|
||||
self.sources["sigma_source"],
|
||||
prefix="sigma",
|
||||
tooltips=sigma_boxplot_tooltips,
|
||||
tooltips_formatters=sigma_boxplot_tooltips_formatters)
|
||||
self.doc.add_root(self.plots["sigma_inspect"])
|
||||
|
||||
# s plots
|
||||
self.plots["s10_inspect"] = figure(
|
||||
name="s10_inspect",
|
||||
title="",
|
||||
plot_width=900, plot_height=400, x_range=[""],
|
||||
tools=tools, sizing_mode='scale_width'
|
||||
)
|
||||
plot.fill_boxplot(
|
||||
self.plots["s10_inspect"],
|
||||
self.sources["s10_source"],
|
||||
prefix="s10",
|
||||
tooltips=s10_boxplot_tooltips,
|
||||
tooltips_formatters=s10_boxplot_tooltips_formatters)
|
||||
s10_tab_inspect = Panel(
|
||||
child=self.plots["s10_inspect"],
|
||||
title="Base 10")
|
||||
|
||||
self.plots["s2_inspect"] = figure(
|
||||
name="s2_inspect",
|
||||
title="",
|
||||
plot_width=900, plot_height=400, x_range=[""],
|
||||
tools=tools, sizing_mode='scale_width'
|
||||
)
|
||||
plot.fill_boxplot(
|
||||
self.plots["s2_inspect"], self.sources["s2_source"], prefix="s2",
|
||||
tooltips=s2_boxplot_tooltips,
|
||||
tooltips_formatters=s2_boxplot_tooltips_formatters
|
||||
)
|
||||
s2_tab_inspect = Panel(child=self.plots["s2_inspect"], title="Base 2")
|
||||
|
||||
s_tabs_inspect = Tabs(
|
||||
name="s_tabs_inspect",
|
||||
tabs=[s10_tab_inspect, s2_tab_inspect], tabs_location="below"
|
||||
)
|
||||
self.doc.add_root(s_tabs_inspect)
|
||||
|
||||
def setup_widgets(self):
|
||||
|
||||
# Generation of selectable items
|
||||
|
||||
# Dict contains all inspectable runs (maps display strings to timestamps)
|
||||
# The dict structure allows to get the timestamp from the display string
|
||||
# in O(1)
|
||||
self.runs_dict = self.gen_runs_selection()
|
||||
|
||||
# Dict maps display strings to column names for the different factors
|
||||
# (var, backend, test)
|
||||
self.factors_dict = {
|
||||
"Variables": "variable",
|
||||
"Backends": "vfc_backend",
|
||||
"Tests": "test"
|
||||
}
|
||||
|
||||
# Run selection
|
||||
|
||||
# Contains all options strings
|
||||
runs_display = list(self.runs_dict.keys())
|
||||
# Will be used when updating plots (contains actual number)
|
||||
self.current_run = self.runs_dict[runs_display[-1]]
|
||||
# Contains the selected option string, used to update current_n_runs
|
||||
current_run_display = runs_display[-1]
|
||||
# This contains only entries matching the run
|
||||
self.run_data = self.data[self.data["timestamp"] == self.current_run]
|
||||
|
||||
change_run_callback_js = "updateRunMetadata(cb_obj.value);"
|
||||
|
||||
self.widgets["select_run"] = Select(
|
||||
name="select_run", title="Run :",
|
||||
value=current_run_display, options=runs_display
|
||||
)
|
||||
self.doc.add_root(self.widgets["select_run"])
|
||||
self.widgets["select_run"].on_change("value", self.update_run)
|
||||
self.widgets["select_run"].js_on_change("value", CustomJS(
|
||||
code=change_run_callback_js,
|
||||
args=(dict(
|
||||
metadata=helper.metadata_to_dict(
|
||||
helper.get_metadata(self.metadata, self.current_run)
|
||||
)
|
||||
))
|
||||
))
|
||||
|
||||
# Factors selection
|
||||
|
||||
# "Group by" radio
|
||||
self.widgets["groupby_radio"] = RadioButtonGroup(
|
||||
name="groupby_radio",
|
||||
labels=list(self.factors_dict.keys()), active=0
|
||||
)
|
||||
self.doc.add_root(self.widgets["groupby_radio"])
|
||||
# The functions are defined inside the template to avoid writing too
|
||||
# much JS server side
|
||||
self.widgets["groupby_radio"].on_change(
|
||||
"active",
|
||||
self.update_groupby
|
||||
)
|
||||
|
||||
# "Filter by" radio
|
||||
# Get all possible factors, and remove the one selected in "Group by"
|
||||
filterby_list = list(self.factors_dict.keys())
|
||||
del filterby_list[self.widgets["groupby_radio"].active]
|
||||
|
||||
self.widgets["filterby_radio"] = RadioButtonGroup(
|
||||
name="filterby_radio",
|
||||
labels=filterby_list, active=0
|
||||
)
|
||||
self.doc.add_root(self.widgets["filterby_radio"])
|
||||
# The functions are defined inside the template to avoid writing too
|
||||
# much JS server side
|
||||
self.widgets["filterby_radio"].on_change(
|
||||
"active",
|
||||
self.update_filterby
|
||||
)
|
||||
|
||||
# Filter selector
|
||||
|
||||
filterby = self.widgets["filterby_radio"].labels[
|
||||
self.widgets["filterby_radio"].active
|
||||
]
|
||||
filterby = self.factors_dict[filterby]
|
||||
|
||||
options = self.run_data.index\
|
||||
.get_level_values(filterby).drop_duplicates().tolist()
|
||||
|
||||
self.widgets["select_filter"] = Select(
|
||||
# We need a different name to avoid collision in the template with
|
||||
# the runs comparison's widget
|
||||
name="select_filter", title="Select a filter :",
|
||||
value=options[0], options=options
|
||||
)
|
||||
self.doc.add_root(self.widgets["select_filter"])
|
||||
self.widgets["select_filter"]\
|
||||
.on_change("value", self.update_filter)
|
||||
|
||||
# Toggle for outliers filtering
|
||||
|
||||
self.widgets["outliers_filtering_inspect"] = CheckboxGroup(
|
||||
name="outliers_filtering_inspect",
|
||||
labels=["Filter outliers"], active=[]
|
||||
)
|
||||
self.doc.add_root(self.widgets["outliers_filtering_inspect"])
|
||||
self.widgets["outliers_filtering_inspect"]\
|
||||
.on_change("active", self.update_outliers_filtering)
|
||||
|
||||
# Communication methods
|
||||
# (to send/receive messages to/from master)
|
||||
|
||||
# When received, switch to the run_name in parameter
|
||||
|
||||
def switch_view(self, run_name):
|
||||
self.widgets["select_run"].value = run_name
|
||||
|
||||
# Constructor
|
||||
|
||||
def __init__(self, master, doc, data, metadata):
|
||||
|
||||
self.master = master
|
||||
|
||||
self.doc = doc
|
||||
self.data = data
|
||||
self.metadata = metadata
|
||||
|
||||
self.sources = {
|
||||
"mu_source": ColumnDataSource(data={}),
|
||||
"sigma_source": ColumnDataSource(data={}),
|
||||
"s10_source": ColumnDataSource(data={}),
|
||||
"s2_source": ColumnDataSource(data={})
|
||||
}
|
||||
|
||||
self.plots = {}
|
||||
self.widgets = {}
|
||||
|
||||
# Setup Bokeh objects
|
||||
self.setup_plots()
|
||||
self.setup_widgets()
|
||||
|
||||
# Pass the initial metadata to the template (will be updated in CustomJS
|
||||
# callbacks). This is required because metadata is not displayed in a
|
||||
# Bokeh widget, so we can't update this with a server callback.
|
||||
initial_run = helper.get_metadata(self.metadata, self.current_run)
|
||||
self.doc.template_variables["initial_timestamp"] = self.current_run
|
||||
|
||||
# At this point, everything should have been initialized, so we can
|
||||
# show the plots for the first time
|
||||
self.update_plots()
|
||||
@ -1,208 +0,0 @@
|
||||
# Look for and read all the run files in the current directory (ending with
|
||||
# .vfcrunh5), and lanch a Bokeh server for the visualization of this data.
|
||||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
|
||||
import pandas as pd
|
||||
|
||||
from bokeh.plotting import curdoc
|
||||
|
||||
# Local imports from vfc_ci_server
|
||||
import compare_runs
|
||||
import inspect_runs
|
||||
import helper
|
||||
|
||||
##########################################################################
|
||||
|
||||
# Read vfcrun files, and aggregate them in one dataset
|
||||
|
||||
run_files = [f for f in os.listdir(".") if f.endswith(".vfcrun.h5")]
|
||||
|
||||
if len(run_files) == 0:
|
||||
print(
|
||||
"Warning [vfc_ci]: Could not find any vfcrun files in the directory. "
|
||||
"This will result in server errors and prevent you from viewing the report.")
|
||||
|
||||
# These are arrays of Pandas dataframes for now
|
||||
metadata = []
|
||||
data = []
|
||||
|
||||
for f in run_files:
|
||||
metadata.append(pd.read_hdf(f, "metadata"))
|
||||
data.append(pd.read_hdf(f, "data"))
|
||||
|
||||
metadata = pd.concat(metadata).sort_index()
|
||||
data = pd.concat(data).sort_index()
|
||||
|
||||
|
||||
# Generate the display strings for runs (runs ticks)
|
||||
# By doing this in master, we ensure the homogeneity of display strings
|
||||
# across all plots
|
||||
metadata["name"] = metadata.index.to_series().map(
|
||||
lambda x: helper.get_run_name(
|
||||
x,
|
||||
helper.get_metadata(metadata, x)["hash"]
|
||||
)
|
||||
)
|
||||
helper.reset_run_strings()
|
||||
|
||||
metadata["date"] = metadata.index.to_series().map(
|
||||
lambda x: time.ctime(x)
|
||||
)
|
||||
|
||||
|
||||
##########################################################################
|
||||
|
||||
|
||||
curdoc().title = "Verificarlo Report"
|
||||
|
||||
# Read server arguments
|
||||
# (this is quite easy because Bokeh server is called through a wrapper, so
|
||||
# we know exactly what the arguments might be)
|
||||
|
||||
git_repo_linked = False
|
||||
commit_link = ""
|
||||
|
||||
has_logo = False
|
||||
logo_url = ""
|
||||
|
||||
for i in range(1, len(sys.argv)):
|
||||
|
||||
# Look for the Git repository remote address
|
||||
# (if a Git repo is specified, the webpage will contain hyperlinks to the
|
||||
# repository and the different commits)
|
||||
if sys.argv[i] == "git":
|
||||
from urllib.parse import urlparse
|
||||
|
||||
method = sys.argv[i + 1]
|
||||
address = sys.argv[i + 2]
|
||||
url = ""
|
||||
|
||||
# Here, address is either the remote URL or the path to the local Git
|
||||
# repo (depending on the method)
|
||||
|
||||
if method == "url":
|
||||
# We should directly have a Git URL
|
||||
url = address
|
||||
|
||||
elif method == "directory":
|
||||
# Get the remote URL from the local repo
|
||||
from git import Repo
|
||||
repo = Repo(address)
|
||||
url = repo.remotes.origin.url
|
||||
|
||||
else:
|
||||
raise ValueError(
|
||||
"Error [vfc_ci]: The specified method to get the Git "
|
||||
"repository is invalid. Are you calling Bokeh directly "
|
||||
"instead of using the Verificarlo wrapper ?"
|
||||
)
|
||||
|
||||
# At this point, "url" should be set correctly, we can get the repo's
|
||||
# URL and name, after making sure we're on a Git URL
|
||||
|
||||
parsed_url = urlparse(url)
|
||||
|
||||
path = parsed_url.path.split("/")
|
||||
if len(path) < 3:
|
||||
raise ValueError(
|
||||
"Error [vfc_ci]: The found URL doesn't seem to be pointing "
|
||||
"to a Git repository (path is too short)"
|
||||
)
|
||||
|
||||
repo_name = path[2]
|
||||
|
||||
curdoc().template_variables["repo_url"] = url
|
||||
curdoc().template_variables["repo_name"] = repo_name
|
||||
|
||||
# We should have a "github.com" or a "*gitlab*" URL
|
||||
|
||||
if parsed_url.netloc == "github.com":
|
||||
commit_link = "https://%s%s/commit/" \
|
||||
% (parsed_url.netloc, parsed_url.path)
|
||||
|
||||
curdoc().template_variables["commit_link"] = commit_link
|
||||
curdoc().template_variables["git_host"] = "GitHub"
|
||||
|
||||
# Used in Bokeh tooltips
|
||||
commit_link = commit_link + "@hash"
|
||||
|
||||
# We assume we have a GitLab URL
|
||||
else:
|
||||
commit_link = "https://%s%s/-/commit/" \
|
||||
% (parsed_url.netloc, parsed_url.path)
|
||||
|
||||
curdoc().template_variables["commit_link"] = commit_link
|
||||
curdoc().template_variables["git_host"] = "GitLab"
|
||||
|
||||
# Used in Bokeh tooltips
|
||||
commit_link = commit_link + "@hash"
|
||||
|
||||
git_repo_linked = True
|
||||
|
||||
# Look for a logo URL
|
||||
# If a logo URL is specified, it will be included in the report's header
|
||||
if sys.argv[i] == "logo":
|
||||
curdoc().template_variables["logo_url"] = sys.argv[i + 1]
|
||||
has_logo = True
|
||||
|
||||
|
||||
# After the loop, we know if a repo has been linked, if we have a logo, ...
|
||||
curdoc().template_variables["git_repo_linked"] = git_repo_linked
|
||||
curdoc().template_variables["has_logo"] = has_logo
|
||||
|
||||
|
||||
##########################################################################
|
||||
|
||||
# Setup report views
|
||||
|
||||
# Define a ViewsMaster class to allow two-ways communication between views.
|
||||
# This approach by classes allows us to have separate scopes for each view and
|
||||
# will be useful if we want to add new views at some point in the future
|
||||
# (instead of having n views with n-1 references each).
|
||||
|
||||
class ViewsMaster:
|
||||
|
||||
# Communication functions
|
||||
|
||||
def go_to_inspect(self, run_name):
|
||||
self.inspect.switch_view(run_name)
|
||||
|
||||
# Constructor
|
||||
|
||||
def __init__(self, data, metadata, git_repo_linked, commit_link):
|
||||
|
||||
self.data = data
|
||||
self.metadata = metadata
|
||||
self.git_repo_linked = git_repo_linked
|
||||
self.commit_link = commit_link
|
||||
|
||||
# Pass metadata to the template as a JSON string
|
||||
curdoc().template_variables["metadata"] = self.metadata.to_json(
|
||||
orient="index")
|
||||
|
||||
# Runs comparison
|
||||
self.compare = compare_runs.CompareRuns(
|
||||
master=self,
|
||||
doc=curdoc(),
|
||||
data=data,
|
||||
metadata=metadata,
|
||||
)
|
||||
|
||||
# Runs inspection
|
||||
self.inspect = inspect_runs.InspectRuns(
|
||||
master=self,
|
||||
doc=curdoc(),
|
||||
data=data,
|
||||
metadata=metadata,
|
||||
)
|
||||
|
||||
|
||||
views_master = ViewsMaster(
|
||||
data=data,
|
||||
metadata=metadata,
|
||||
git_repo_linked=git_repo_linked,
|
||||
commit_link=commit_link
|
||||
)
|
||||
@ -1,142 +0,0 @@
|
||||
# General functions for filling plots with data in all report's views
|
||||
|
||||
from bokeh.plotting import figure
|
||||
from bokeh.models import HoverTool, TapTool, CustomJS
|
||||
|
||||
from math import pi
|
||||
|
||||
|
||||
def fill_dotplot(
|
||||
plot, source, data_field,
|
||||
tooltips=None, tooltips_formatters=None,
|
||||
js_tap_callback=None, server_tap_callback=None,
|
||||
lines=False,
|
||||
lower_bound=False
|
||||
):
|
||||
|
||||
# (Optional) Tooltip and tooltip formatters
|
||||
if tooltips is not None:
|
||||
hover = HoverTool(tooltips=tooltips, mode="vline", names=["circle"])
|
||||
|
||||
if tooltips_formatters is not None:
|
||||
hover.formatters = tooltips_formatters
|
||||
|
||||
plot.add_tools(hover)
|
||||
|
||||
# (Optional) Add TapTool (for JS tap callback)
|
||||
if js_tap_callback is not None:
|
||||
tap = TapTool(callback=CustomJS(code=js_tap_callback))
|
||||
plot.add_tools(tap)
|
||||
|
||||
# (Optional) Add segment to represent a lower bound
|
||||
if lower_bound:
|
||||
lower_segment = plot.segment(
|
||||
x0="%s_x" % data_field, y0=data_field,
|
||||
x1="%s_x" % data_field, y1="%s_lower_bound" % data_field,
|
||||
source=source, line_color="black"
|
||||
)
|
||||
|
||||
# Draw dots (actually Bokeh circles)
|
||||
circle = plot.circle(
|
||||
name="circle",
|
||||
x="%s_x" % data_field, y=data_field, source=source, size=12
|
||||
)
|
||||
|
||||
# (Optional) Draw lines between dots
|
||||
if lines:
|
||||
line = plot.line(x="%s_x" % data_field, y=data_field, source=source)
|
||||
|
||||
# (Optional) Add server tap callback
|
||||
if server_tap_callback is not None:
|
||||
circle.data_source.selected.on_change("indices", server_tap_callback)
|
||||
|
||||
# Plot appearance
|
||||
plot.xgrid.grid_line_color = None
|
||||
plot.ygrid.grid_line_color = None
|
||||
|
||||
plot.yaxis[0].formatter.power_limit_high = 0
|
||||
plot.yaxis[0].formatter.power_limit_low = 0
|
||||
plot.yaxis[0].formatter.precision = 3
|
||||
|
||||
plot.xaxis[0].major_label_orientation = pi / 8
|
||||
|
||||
|
||||
def fill_boxplot(
|
||||
plot, source,
|
||||
prefix="",
|
||||
tooltips=None, tooltips_formatters=None,
|
||||
js_tap_callback=None, server_tap_callback=None
|
||||
):
|
||||
|
||||
# (Optional) Tooltip and tooltip formatters
|
||||
if tooltips is not None:
|
||||
hover = HoverTool(tooltips=tooltips, mode="vline", names=["full_box"])
|
||||
|
||||
if tooltips_formatters is not None:
|
||||
hover.formatters = tooltips_formatters
|
||||
|
||||
plot.add_tools(hover)
|
||||
|
||||
# (Optional) Add TapTool (for JS tap callback)
|
||||
if js_tap_callback is not None:
|
||||
tap = TapTool(callback=CustomJS(code=js_tap_callback))
|
||||
plot.add_tools(tap)
|
||||
|
||||
# Draw boxes (the prefix argument modifies the fields of ColumnDataSource
|
||||
# that are used)
|
||||
|
||||
if prefix != "":
|
||||
prefix = "%s_" % prefix
|
||||
|
||||
# Stems
|
||||
top_stem = plot.segment(
|
||||
x0="%sx" % prefix, y0="%smax" % prefix,
|
||||
x1="%sx" % prefix, y1="%squantile75" % prefix,
|
||||
source=source, line_color="black"
|
||||
)
|
||||
bottom_stem = plot.segment(
|
||||
x0="%sx" % prefix, y0="%smin" % prefix,
|
||||
x1="%sx" % prefix, y1="%squantile25" % prefix,
|
||||
source=source, line_color="black"
|
||||
)
|
||||
|
||||
# Boxes
|
||||
full_box = plot.vbar(
|
||||
name="full_box",
|
||||
x="%sx" % prefix, width=0.5,
|
||||
top="%squantile75" % prefix, bottom="%squantile25" % prefix,
|
||||
source=source, line_color="black"
|
||||
)
|
||||
bottom_box = plot.vbar(
|
||||
x="%sx" % prefix, width=0.5,
|
||||
top="%squantile50" % prefix, bottom="%squantile25" % prefix,
|
||||
source=source, line_color="black"
|
||||
)
|
||||
|
||||
# Mu dot
|
||||
mu_dot = plot.dot(
|
||||
x="%sx" % prefix, y="%smu" % prefix, size=30, source=source,
|
||||
color="black"
|
||||
)
|
||||
|
||||
# (Optional) Add server tap callback
|
||||
if server_tap_callback is not None:
|
||||
top_stem.data_source.selected.on_change("indices", server_tap_callback)
|
||||
bottom_stem.data_source.selected.on_change(
|
||||
"indices", server_tap_callback)
|
||||
|
||||
full_box.data_source.selected.on_change("indices", server_tap_callback)
|
||||
bottom_box.data_source.selected.on_change(
|
||||
"indices", server_tap_callback)
|
||||
|
||||
mu_dot.data_source.selected.on_change("indices", server_tap_callback)
|
||||
|
||||
# Plot appearance
|
||||
plot.xgrid.grid_line_color = None
|
||||
plot.ygrid.grid_line_color = None
|
||||
|
||||
plot.yaxis[0].formatter.power_limit_high = 0
|
||||
plot.yaxis[0].formatter.power_limit_low = 0
|
||||
plot.yaxis[0].formatter.precision = 3
|
||||
|
||||
plot.xaxis[0].major_label_orientation = pi / 8
|
||||
@ -1,482 +0,0 @@
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
|
||||
<head>
|
||||
<title>Verificarlo Report</title>
|
||||
<meta charset="utf8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
|
||||
<!-- This template uses Bulma for CSS : https://bulma.io/ -->
|
||||
<!-- Doc : https://bulma.io/documentation/ -->
|
||||
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bulma@0.9.2/css/bulma.min.css">
|
||||
|
||||
<style>
|
||||
html, body{
|
||||
background-color: #f5f5f5;
|
||||
}
|
||||
|
||||
#navbar {
|
||||
height: 67px;
|
||||
}
|
||||
|
||||
#logo-link {
|
||||
padding: 0;
|
||||
padding-left: 16px;
|
||||
}
|
||||
|
||||
#logo-img {
|
||||
object-fit: cover;
|
||||
margin-top: -18px;
|
||||
margin-left: -8px;
|
||||
max-height: 50px;
|
||||
}
|
||||
|
||||
#compare-runs-container {
|
||||
margin-top: 1em;
|
||||
}
|
||||
|
||||
#inspect-runs-container {
|
||||
margin-top: 1em;
|
||||
display : none; /* This one is hidden by default */
|
||||
}
|
||||
|
||||
.plot-card {
|
||||
width: 900px;
|
||||
}
|
||||
|
||||
#loader {
|
||||
height: 100%;
|
||||
}
|
||||
|
||||
#loading-logo {
|
||||
position: absolute;
|
||||
left: 50%;
|
||||
top: 50%;
|
||||
transform: translate(-50%, -50%);
|
||||
max-width: 400px;
|
||||
|
||||
animation: pulse 1.5s linear infinite;
|
||||
}
|
||||
|
||||
@keyframes pulse {
|
||||
0% {
|
||||
transform: scale(1) translate(-50%, -50%);
|
||||
}
|
||||
|
||||
50% {
|
||||
transform: scale(0.9) translate(-55%, -55%);
|
||||
}
|
||||
|
||||
100% {
|
||||
transform: scale(1) translate(-50%, -50%);
|
||||
}
|
||||
}
|
||||
|
||||
</style>
|
||||
|
||||
{% extends base %}
|
||||
</head>
|
||||
|
||||
|
||||
<body>
|
||||
{% block contents %}
|
||||
|
||||
|
||||
<!-- REPORT -->
|
||||
<div id="report" style="display: none;">
|
||||
|
||||
<!-- HEADER -->
|
||||
<nav class="navbar has-shadow" id="navbar"
|
||||
role="navigation" aria-label="navbar-content">
|
||||
|
||||
<!-- BRAND (left part) -->
|
||||
<div class="navbar-brand">
|
||||
{% if has_logo %}
|
||||
<a
|
||||
class="navbar-item" id="logo-link" href="."
|
||||
style="margin-top: 12px;"
|
||||
>
|
||||
<img id="logo-img" width="85" height="45"
|
||||
src="{{ logo_url }}">
|
||||
</a>
|
||||
{% endif %}
|
||||
|
||||
<a class="navbar-item" id="logo-link" href=".">
|
||||
<img id="logo-img" width="85" height="45"
|
||||
src="https://avatars1.githubusercontent.com/u/12033642">
|
||||
</a>
|
||||
|
||||
<a role="button" class="navbar-burger" id="navbar-burger"
|
||||
aria-label="menu" aria-expanded="false"
|
||||
data-target="navbarBasicExample">
|
||||
<span aria-hidden="true"></span>
|
||||
<span aria-hidden="true"></span>
|
||||
<span aria-hidden="true"></span>
|
||||
</a>
|
||||
</div>
|
||||
|
||||
<!-- MENU (content) -->
|
||||
<div id="navbar-content" class="navbar-menu">
|
||||
|
||||
<div id="buttons-container" class="navbar-start">
|
||||
<a class="navbar-item is-active" id="compare-runs-button">
|
||||
Compare runs
|
||||
</a>
|
||||
|
||||
<a class="navbar-item" id="inspect-runs-button">
|
||||
Inspect runs
|
||||
</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-end">
|
||||
{% if git_repo_linked %}
|
||||
<div class="navbar-item">
|
||||
<div class="buttons">
|
||||
<a class="button is-light" href="{{repo_url}}" target="_blank">
|
||||
{{repo_name}} on {{git_host}}
|
||||
</a>
|
||||
</div>
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
<div class="navbar-item">
|
||||
<div class="buttons">
|
||||
<a class="button is-light"
|
||||
href="https://github.com/verificarlo" target="_blank">
|
||||
Verificarlo on GitHub
|
||||
</a>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</nav>
|
||||
|
||||
|
||||
<!-- CONTENT : COMPARE RUNS -->
|
||||
<main class="container" id="compare-runs-container">
|
||||
<div class="columns">
|
||||
|
||||
<!-- SELECTORS -->
|
||||
<div class="column">
|
||||
<h4 class="title is-4">Selectors</h4>
|
||||
<div id="compare-widgets">
|
||||
{{ embed(roots.test_filter) }}
|
||||
{{ embed(roots.select_test) }}
|
||||
<br>
|
||||
{{ embed(roots.select_var) }}
|
||||
<br>
|
||||
{{ embed(roots.select_backend) }}
|
||||
<br>
|
||||
{{ embed(roots.outliers_filtering_compare) }}
|
||||
<br>
|
||||
<br>
|
||||
{{ embed(roots.select_n_runs) }}
|
||||
|
||||
|
||||
<br>
|
||||
<br>
|
||||
|
||||
<b>Tip :</b> You can click on any element of the plots
|
||||
to inspect the corresponding run in details.
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="is-divider-vertical"></div>
|
||||
|
||||
|
||||
<!-- PLOTS -->
|
||||
<div class="column is-9">
|
||||
<h3 class="title is-3">Plots</h3>
|
||||
<div class="container">
|
||||
<div class="card plot-card">
|
||||
{{ embed(roots.s_tabs) }}
|
||||
</div>
|
||||
|
||||
<br>
|
||||
|
||||
<div class="card plot-card">
|
||||
{{ embed(roots.sigma_plot) }}
|
||||
</div>
|
||||
|
||||
<br>
|
||||
|
||||
<div class="card plot-card">
|
||||
{{ embed(roots.boxplot) }}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</main>
|
||||
|
||||
|
||||
<!-- CONTENT : INSPECT RUNS -->
|
||||
<main class="container" id="inspect-runs-container">
|
||||
<div class="columns">
|
||||
|
||||
<!-- SELECTORS -->
|
||||
<div class="column">
|
||||
<h4 class="title is-4">Selectors</h4>
|
||||
{{ embed(roots.select_run) }}
|
||||
|
||||
<br>
|
||||
|
||||
Group by :
|
||||
{{ embed(roots.groupby_radio) }}
|
||||
|
||||
<br>
|
||||
|
||||
Filter by :
|
||||
{{ embed(roots.filterby_radio) }}
|
||||
{{ embed(roots.select_filter) }}
|
||||
|
||||
<br>
|
||||
|
||||
{{ embed(roots.outliers_filtering_inspect) }}
|
||||
|
||||
<br>
|
||||
<br>
|
||||
|
||||
<h4 class="title is-4">Run metadata</h4>
|
||||
|
||||
<b>Date :</b>
|
||||
<div id="run-date" style="display: inline;">
|
||||
</div>
|
||||
|
||||
<br>
|
||||
|
||||
<div id="is-git-commit">
|
||||
|
||||
<b>Hash :</b>
|
||||
<div
|
||||
id="run-hash" style="display: inline;">
|
||||
</div>
|
||||
<br>
|
||||
<b>Author :</b>
|
||||
<div id="run-author" style="display: inline;">
|
||||
</div>
|
||||
<br>
|
||||
<b>Message :</b>
|
||||
<div id="run-message" style="display: inline;">
|
||||
</div>
|
||||
|
||||
{% if git_repo_linked %}
|
||||
|
||||
<br>
|
||||
<br>
|
||||
|
||||
|
||||
<a
|
||||
id="git-commit-link"
|
||||
href=""
|
||||
target="_blank"
|
||||
>
|
||||
View this commit on {{git_host}}
|
||||
</a>
|
||||
|
||||
{% endif %}
|
||||
|
||||
</div>
|
||||
|
||||
<div id="not-git-commit">
|
||||
|
||||
This run is not linked to a Git commit.
|
||||
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
||||
|
||||
<!-- PLOTS -->
|
||||
<div class="column is-9">
|
||||
<h3 class="title is-3">Plots</h3>
|
||||
<div class="card plot-card" style="z-index: 3;">
|
||||
{{ embed(roots.s_tabs_inspect) }}
|
||||
</div>
|
||||
|
||||
<br>
|
||||
|
||||
<div class="card plot-card" style="z-index: 2;">
|
||||
{{ embed(roots.sigma_inspect) }}
|
||||
</div>
|
||||
|
||||
<br>
|
||||
|
||||
<div class="card plot-card" style="z-index: 1;">
|
||||
{{ embed(roots.mu_inspect) }}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</main>
|
||||
|
||||
</div>
|
||||
|
||||
|
||||
<!--- LOADER -->
|
||||
<div id="loader">
|
||||
|
||||
{% if has_logo %}
|
||||
<img id="loading-logo" src="{{logo_url}}">
|
||||
{% else %}
|
||||
<img id="loading-logo" src="https://avatars1.githubusercontent.com/u/12033642">
|
||||
{% endif %}
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
<!-- JAVASCRIPT -->
|
||||
<script>
|
||||
|
||||
// Listen to clicks on breadcrumb (for responsive header)
|
||||
document.getElementById("navbar-burger")
|
||||
.addEventListener("click", () => {
|
||||
|
||||
document.getElementById("navbar-burger")
|
||||
.classList.toggle("is-active");
|
||||
|
||||
document.getElementById("navbar-content")
|
||||
.classList.toggle("is-active");
|
||||
|
||||
})
|
||||
|
||||
|
||||
|
||||
// Helper function to navigate between views
|
||||
function changeView(classPrefix) {
|
||||
|
||||
// Enable/disable the active class on buttons
|
||||
let buttons = document.getElementById("buttons-container")
|
||||
.childNodes;
|
||||
let toggledButtonId = classPrefix + "-button";
|
||||
|
||||
for(let i=0; i<buttons.length; i++) {
|
||||
console.log()
|
||||
if(toggledButtonId == buttons[i].id) {
|
||||
buttons[i].classList.add("is-active");
|
||||
}
|
||||
else if(buttons[i].classList != undefined) {
|
||||
buttons[i].classList.remove("is-active");
|
||||
}
|
||||
}
|
||||
|
||||
// Show hide the containers
|
||||
let containers = document.getElementsByTagName("MAIN");
|
||||
let toggledContainerId = classPrefix + "-container"
|
||||
|
||||
for(let i=0; i<containers.length; i++) {
|
||||
if(toggledContainerId == containers[i].id) {
|
||||
containers[i].style.display = "block";
|
||||
}
|
||||
else {
|
||||
containers[i].style.display = "none";
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Listen to clicks on "Compare runs" button
|
||||
document.getElementById("compare-runs-button")
|
||||
.addEventListener("click", () => {
|
||||
// Nothing else to do for this button
|
||||
changeView("compare-runs");
|
||||
});
|
||||
|
||||
// Listen to clicks on "Inspect runs" button
|
||||
// (dedicated function as this needs to be called in a CustomJS callback)
|
||||
function goToInspectRuns() {
|
||||
window.scrollTo(0, 0);
|
||||
|
||||
changeView("inspect-runs");
|
||||
}
|
||||
|
||||
document.getElementById("inspect-runs-button")
|
||||
.addEventListener("click", goToInspectRuns);
|
||||
|
||||
|
||||
|
||||
// Toggle the display properties of the loader/report
|
||||
function removeLoader() {
|
||||
document.getElementById("loader")
|
||||
.style.display = "none";
|
||||
|
||||
document.getElementById("report")
|
||||
.style.display = "";
|
||||
}
|
||||
|
||||
// To detect the end of Bokeh initialization and remove the loader,
|
||||
// we look at the number of children of a div containing widgets
|
||||
let nChildren = document.getElementById('compare-widgets')
|
||||
.getElementsByTagName('*').length;
|
||||
|
||||
function pollBokehLoading() {
|
||||
let newNChildren = document.getElementById('compare-widgets')
|
||||
.getElementsByTagName('*').length;
|
||||
|
||||
if(newNChildren != nChildren) {
|
||||
removeLoader();
|
||||
}
|
||||
else {
|
||||
setTimeout(pollBokehLoading, 100);
|
||||
}
|
||||
}
|
||||
setTimeout(pollBokehLoading, 100);
|
||||
|
||||
|
||||
|
||||
// Update the run metadata (in inspect run mode)
|
||||
function updateRunMetadata(runId) {
|
||||
|
||||
// Assume runId is the run's timestamp
|
||||
let run = metadata[runId];
|
||||
|
||||
// If it is undefined, perform a search by name
|
||||
// (by iterating metadata)
|
||||
if(!run) {
|
||||
for(let [key, value] of Object.entries(metadata)) {
|
||||
|
||||
if (!metadata.hasOwnProperty(key)) continue;
|
||||
if(value.name == runId) {
|
||||
run = value;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
document.getElementById("run-date").innerHTML = run.date;
|
||||
|
||||
if(run.is_git_commit) {
|
||||
document.getElementById("is-git-commit").style.display = "";
|
||||
document.getElementById("not-git-commit").style.display = "none";
|
||||
|
||||
document.getElementById("run-hash").innerHTML = run.hash;
|
||||
document.getElementById("run-author").innerHTML = run.author;
|
||||
document.getElementById("run-message").innerHTML = run.message;
|
||||
|
||||
{% if git_repo_linked %}
|
||||
document.getElementById("git-commit-link")
|
||||
.setAttribute("href", "{{commit_link}}" + run.hash);
|
||||
{% endif %}
|
||||
|
||||
} else {
|
||||
document.getElementById("is-git-commit").style.display = "none";
|
||||
document.getElementById("not-git-commit").style.display = "";
|
||||
|
||||
document.getElementById("run-hash").innerHTML = "";
|
||||
document.getElementById("run-author").innerHTML = "";
|
||||
document.getElementById("run-message").innerHTML = "";
|
||||
}
|
||||
}
|
||||
|
||||
//Object containing metadata from all runs
|
||||
metadata = {{metadata}}
|
||||
|
||||
// Initial run using the template arg
|
||||
updateRunMetadata({{initial_timestamp}});
|
||||
|
||||
</script>
|
||||
|
||||
{% endblock %}
|
||||
</body>
|
||||
|
||||
</html>
|
||||
@ -1,7 +0,0 @@
|
||||
## Verificarlo CI : `{{dev_branch}}`
|
||||
|
||||
You are on the `{{ci_branch}}` branch, which is automatically updated with the
|
||||
[Verificarlo](https://github.com/verificarlo/verificarlo) test results from
|
||||
`{{dev_branch}}` (in the `vfcruns` directory).
|
||||
|
||||
You can start a Verificarlo CI server at anytime using the run files of this branch.
|
||||
@ -1,42 +0,0 @@
|
||||
# This workflow will be executed when {{dev_branch}} is updated:
|
||||
# it will run the configured tests and upload the results on vfc_ci_master.
|
||||
|
||||
image: verificarlo/verificarlo
|
||||
|
||||
|
||||
stages:
|
||||
- run_verificarlo_tests
|
||||
|
||||
|
||||
run_verificarlo_tests:
|
||||
stage: run_verificarlo_tests
|
||||
|
||||
before_script:
|
||||
- git remote set-url origin https://{{username}}:${CI_PUSH_TOKEN}@{{remote_url}}.git
|
||||
- git config --global user.email "{{email}}"
|
||||
- git config --global user.name "{{username}}"
|
||||
|
||||
script:
|
||||
# We will probably drop these installations when integrating CI into
|
||||
# Verificarlo
|
||||
- pip install numpy scipy pandas bokeh jinja2 tables GitPython
|
||||
- apt update
|
||||
- apt install wget
|
||||
- wget https://raw.githubusercontent.com/verificarlo/significantdigits/main/sigdigits.py -P /usr/local/lib/python3.8/dist-packages
|
||||
|
||||
- ./vfc_ci test -g -r
|
||||
- git_hash=$(git rev-parse --short "$CI_COMMIT_SHA")
|
||||
- git fetch --all
|
||||
- git checkout -b {{ci_branch}} origin/{{ci_branch}}
|
||||
- mkdir -p vfcruns
|
||||
- mv *.vfcrun.h5 vfcruns
|
||||
- git add vfcruns/*
|
||||
- git commit -m "[auto] New test results for commit ${git_hash}"
|
||||
- git push
|
||||
|
||||
rules:
|
||||
- if: '$CI_COMMIT_BRANCH == "{{dev_branch}}"'
|
||||
|
||||
artifacts:
|
||||
paths:
|
||||
- "*.vfcraw.h5"
|
||||
@ -1,57 +0,0 @@
|
||||
# This workflow will be executed when {{dev_branch}} is updated:
|
||||
# it will run the configured tests and upload the results on {{ci_branch}}.
|
||||
|
||||
name: "Verificarlo CI ({{dev_branch}})"
|
||||
|
||||
on:
|
||||
# Triggers the workflow when {{dev_branch}} is updated
|
||||
push:
|
||||
branches: [ {{dev_branch}} ]
|
||||
|
||||
workflow_dispatch:
|
||||
|
||||
|
||||
jobs:
|
||||
run_verificarlo_tests:
|
||||
runs-on: ubuntu-latest
|
||||
container: verificarlo/verificarlo
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
# We will probably drop these installations when integrating CI into
|
||||
# Verificarlo
|
||||
- name: Install Python requirements
|
||||
run: |
|
||||
pip install numpy scipy pandas bokeh jinja2 tables GitPython
|
||||
apt update
|
||||
apt install wget
|
||||
wget https://raw.githubusercontent.com/verificarlo/significantdigits/main/sigdigits.py -P /usr/local/lib/python3.8/dist-packages
|
||||
|
||||
- name: Run tests
|
||||
# We assume the script is included in the repo for now
|
||||
# (we'll probably want to remove "./" if the script ends up being integrated
|
||||
# in Verificarlo and becomes available system-wide)
|
||||
run: ./vfc_ci test -g -r
|
||||
|
||||
- name: Commit test results
|
||||
run: |
|
||||
git_hash=$(git rev-parse --short "$GITHUB_SHA")
|
||||
|
||||
git config --local user.email "action@github.com"
|
||||
git config --local user.name "GitHub Action"
|
||||
|
||||
git checkout {{ci_branch}}
|
||||
mkdir -p vfcruns
|
||||
mv *.vfcrun.h5 vfcruns
|
||||
git add vfcruns/*
|
||||
git commit -m "[auto] New test results for commit ${git_hash}"
|
||||
git push
|
||||
|
||||
- name: Upload raw results as artifacts
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
{% raw %}name: ${{github.sha}}.vfcraw{% endraw %}
|
||||
path: ./*.vfcraw.h5
|
||||
@ -150,26 +150,33 @@ int test_cycle(H5File file, int cycle, std::string version,
|
||||
}
|
||||
|
||||
int main(int argc, char **argv) {
|
||||
if (argc != 3) {
|
||||
if (argc != 2) {
|
||||
std::cerr << "Execute from within '/'" << std::endl;
|
||||
std::cerr << "usage: test_h5 <version> <path to cycles file>" << std::endl;
|
||||
std::cerr << "usage: vfc_test_h5 <path to cycles file>" << std::endl;
|
||||
return 1;
|
||||
}
|
||||
std::string version(argv[1]);
|
||||
std::vector<int> cycles_list = get_cycles_list(argv[2]);
|
||||
std::vector<int> cycles_list = get_cycles_list(argv[1]);
|
||||
H5File file(FILE_NAME, H5F_ACC_RDONLY);
|
||||
|
||||
vfc_probes probes = vfc_init_probes();
|
||||
probes = vfc_init_probes();
|
||||
|
||||
std::vector<std::string> versions = {"maponia3", "sm1", "sm2", "sm3", "sm4"};
|
||||
|
||||
bool ok;
|
||||
for (int i = 0; i < cycles_list.size(); i++) {
|
||||
ok = test_cycle(file, cycles_list[i], version, &probes);
|
||||
if (ok) {
|
||||
std::cout << "ok -- cycle " << std::to_string(i) << std::endl;
|
||||
} else {
|
||||
std::cerr << "failed -- cycle " << std::to_string(i) << std::endl;
|
||||
|
||||
for(const auto& version: versions) {
|
||||
|
||||
ok = test_cycle(file, cycles_list[i], version, &probes);
|
||||
if (ok) {
|
||||
std::cout << "ok -- cycle " << std::to_string(i) << std::endl;
|
||||
} else {
|
||||
std::cerr << "failed -- cycle " << std::to_string(i) << std::endl;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
vfc_dump_probes(&probes);
|
||||
|
||||
199
vfc_ci
199
vfc_ci
@ -1,199 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
# This is the entry point of the Verificarlo CI command line interface, which is
|
||||
# based on argparse and this article :
|
||||
# https://mike.depalatis.net/blog/simplifying-argparse.html
|
||||
# From here, 3 subcommands can be called :
|
||||
# - setup : create a vfc_ci branch and workflow on the current Git repo
|
||||
# - test : run and export test results according to the vfc_tests_config.json
|
||||
# - serve : launch a Bokeh server to visualize run results
|
||||
|
||||
import argparse
|
||||
|
||||
##########################################################################
|
||||
|
||||
# Parameters validation helpers
|
||||
|
||||
|
||||
def is_port(string):
|
||||
value = int(string)
|
||||
if value < 0 or value > 65535:
|
||||
raise argparse.ArgumentTypeError("Value has to be between 0 and 65535")
|
||||
return value
|
||||
|
||||
|
||||
def is_directory(string):
|
||||
import os
|
||||
|
||||
isdir = os.path.isdir(string)
|
||||
if not isdir:
|
||||
raise argparse.ArgumentTypeError("Directory does not exist")
|
||||
|
||||
return string
|
||||
|
||||
|
||||
##########################################################################
|
||||
|
||||
# Subcommand decorator
|
||||
|
||||
cli = argparse.ArgumentParser(
|
||||
description="Define, run, automatize, and visualize custom Verificarlo tests."
|
||||
)
|
||||
subparsers = cli.add_subparsers(dest="subcommand")
|
||||
|
||||
|
||||
def subcommand(description="", args=[], parent=subparsers):
|
||||
def decorator(func):
|
||||
parser = parent.add_parser(func.__name__, description=description)
|
||||
for arg in args:
|
||||
parser.add_argument(*arg[0], **arg[1])
|
||||
parser.set_defaults(func=func)
|
||||
return decorator
|
||||
|
||||
|
||||
def argument(*name_or_flags, **kwargs):
|
||||
return ([*name_or_flags], kwargs)
|
||||
|
||||
|
||||
##########################################################################
|
||||
|
||||
# "setup" subcommand
|
||||
|
||||
@subcommand(
|
||||
description="Create an automated workflow to execute Verificarlo tests.",
|
||||
args=[
|
||||
argument(
|
||||
"git_host",
|
||||
help="""
|
||||
specify where your repository is hosted
|
||||
""",
|
||||
choices=["github", "gitlab"]
|
||||
)
|
||||
]
|
||||
)
|
||||
def setup(args):
|
||||
import ci.setup
|
||||
ci.setup.setup(args.git_host)
|
||||
|
||||
# "test" subcommand
|
||||
|
||||
|
||||
@subcommand(
|
||||
description="Execute predefined Verificarlo tests and save their results.",
|
||||
args=[
|
||||
argument(
|
||||
"-g", "--is-git-commit",
|
||||
help="""
|
||||
When specified, the last Git commit of the local repository (working
|
||||
directory) will be fetched and associated with the run.
|
||||
""",
|
||||
action="store_true"
|
||||
),
|
||||
argument(
|
||||
"-r", "--export-raw-results",
|
||||
help="""
|
||||
Specify if an additional HDF5 file containing the raw results must be
|
||||
exported.
|
||||
""",
|
||||
action="store_true"
|
||||
),
|
||||
argument(
|
||||
"-d", "--dry-run",
|
||||
help="""
|
||||
Perform a dry run by not saving the test results.
|
||||
""",
|
||||
action="store_true"
|
||||
)
|
||||
]
|
||||
)
|
||||
def test(args):
|
||||
import ci.test
|
||||
ci.test.run(args.is_git_commit, args.export_raw_results, args.dry_run)
|
||||
|
||||
# "serve" subcommand
|
||||
|
||||
|
||||
@subcommand(
|
||||
description="""
|
||||
Start a server to visualize Verificarlo test results.
|
||||
""",
|
||||
args=[
|
||||
argument(
|
||||
"-s", "--show",
|
||||
help="""
|
||||
Specify if the report must be opened in the browser at server
|
||||
startup.
|
||||
""",
|
||||
action="store_true"
|
||||
),
|
||||
argument(
|
||||
"-gd", "--git-directory",
|
||||
help="""
|
||||
Path to a local Git repository. The report will be linked to the
|
||||
remote URL (GitHub and GitLab are supported).
|
||||
""",
|
||||
type=is_directory
|
||||
),
|
||||
argument(
|
||||
"-gu", "--git-url",
|
||||
help="""
|
||||
GitHub or GitLab repository URL. The report will be linked to this
|
||||
URL.
|
||||
""",
|
||||
type=str
|
||||
),
|
||||
argument(
|
||||
"-p", "--port",
|
||||
help="""
|
||||
The port on which the server will run. Defaults to 8080.
|
||||
""",
|
||||
type=is_port,
|
||||
default=8080
|
||||
),
|
||||
argument(
|
||||
"-a", "--allow-origin",
|
||||
help="""
|
||||
The origin (URL) from which the report will be accessible.
|
||||
Port number must not be specified. Defaults to * (allow everything).
|
||||
""",
|
||||
type=str,
|
||||
default="*"
|
||||
),
|
||||
argument(
|
||||
"-l", "--logo",
|
||||
help="""
|
||||
Specify the URL of an image to be displayed in the report header.
|
||||
""",
|
||||
type=str
|
||||
)
|
||||
]
|
||||
)
|
||||
def serve(args):
|
||||
|
||||
# git_directory and git_url are supposed to be exclusive
|
||||
if args.git_directory is not None and args.git_url is not None:
|
||||
raise argparse.ArgumentTypeError(
|
||||
"\"-gd\" / \"--git-directory\" and \"-gu\" / \"--git-url\" are "
|
||||
"mutually exclusive. Please make sure to use at most one of them."
|
||||
)
|
||||
|
||||
import ci.serve
|
||||
ci.serve.serve(
|
||||
args.show,
|
||||
args.git_directory,
|
||||
args.git_url,
|
||||
args.port,
|
||||
args.allow_origin,
|
||||
args.logo
|
||||
)
|
||||
|
||||
|
||||
###############################################################################
|
||||
|
||||
# Main command group and entry point
|
||||
if __name__ == "__main__":
|
||||
args = cli.parse_args()
|
||||
if args.subcommand is None:
|
||||
cli.print_help()
|
||||
else:
|
||||
args.func(args)
|
||||
@ -1,69 +1,10 @@
|
||||
|
||||
{
|
||||
"make_command": "make -f Makefile.verificarlo",
|
||||
"executables": [
|
||||
{
|
||||
"executable": "bin/vfc_test_h5",
|
||||
"parameters" : "maponia3 vfc_ci_cycles.txt",
|
||||
"vfc_backends": [
|
||||
{
|
||||
"name": "libinterflop_mca.so",
|
||||
"repetitions": 50
|
||||
},
|
||||
{
|
||||
"name": "libinterflop_mca.so --mode=rr",
|
||||
"repetitions": 50
|
||||
}
|
||||
]
|
||||
},
|
||||
|
||||
{
|
||||
"executable": "bin/vfc_test_h5",
|
||||
"parameters" : "sm1 vfc_ci_cycles.txt",
|
||||
"vfc_backends": [
|
||||
{
|
||||
"name": "libinterflop_mca.so",
|
||||
"repetitions": 50
|
||||
},
|
||||
{
|
||||
"name": "libinterflop_mca.so --mode=rr",
|
||||
"repetitions": 50
|
||||
}
|
||||
]
|
||||
},
|
||||
|
||||
{
|
||||
"executable": "bin/vfc_test_h5",
|
||||
"parameters" : "sm2 vfc_ci_cycles.txt",
|
||||
"vfc_backends": [
|
||||
{
|
||||
"name": "libinterflop_mca.so",
|
||||
"repetitions": 50
|
||||
},
|
||||
{
|
||||
"name": "libinterflop_mca.so --mode=rr",
|
||||
"repetitions": 50
|
||||
}
|
||||
]
|
||||
},
|
||||
|
||||
{
|
||||
"executable": "bin/vfc_test_h5",
|
||||
"parameters" : "sm3 vfc_ci_cycles.txt",
|
||||
"vfc_backends": [
|
||||
{
|
||||
"name": "libinterflop_mca.so",
|
||||
"repetitions": 50
|
||||
},
|
||||
{
|
||||
"name": "libinterflop_mca.so --mode=rr",
|
||||
"repetitions": 50
|
||||
}
|
||||
]
|
||||
},
|
||||
|
||||
{
|
||||
"executable": "bin/vfc_test_h5",
|
||||
"parameters" : "sm4 vfc_ci_cycles.txt",
|
||||
"parameters" : "vfc_ci_cycles.txt",
|
||||
"vfc_backends": [
|
||||
{
|
||||
"name": "libinterflop_mca.so",
|
||||
|
||||
Loading…
Reference in New Issue
Block a user