mirror of
https://github.com/TREX-CoE/Sherman-Morrison.git
synced 2024-12-25 13:53:56 +01:00
Remove source code of vfc_ci and improve vfc_test_h5.cpp
Since vfc_ci has been merged wih the main verificarlo repository and is now available in Verificarlo's Docker, the source code of vfc_ci can now be removed. Also, vfc_test_h5.cpp has been modified so that every algorithm is called directly. This results in a much shorter vfc_tests_config.json since we don't need to vall vfc_test_h5 with all algorithms as arguments.
This commit is contained in:
Aurélien Delval
parent
cb8cb2c426
commit
0114fd7b14
15
.github/workflows/vfc_test_workflow.yml
vendored
15
.github/workflows/vfc_test_workflow.yml
vendored
@ -21,35 +21,24 @@ jobs:
|
|||||||
with:
|
with:
|
||||||
fetch-depth: 0
|
fetch-depth: 0
|
||||||
|
|
||||||
# We will probably drop these installations when integrating CI into
|
- name: Install HDF5 requirements
|
||||||
# Verificarlo
|
|
||||||
- name: Install Python requirements
|
|
||||||
run: |
|
run: |
|
||||||
pip install numpy scipy pandas bokeh jinja2 tables GitPython
|
|
||||||
apt update
|
apt update
|
||||||
apt install -y wget libhdf5-dev g++
|
apt install -y wget libhdf5-dev g++
|
||||||
wget https://raw.githubusercontent.com/verificarlo/significantdigits/main/sigdigits.py -P /usr/local/lib/python3.8/dist-packages
|
|
||||||
|
|
||||||
- name: Run tests
|
- name: Run tests
|
||||||
# We assume the script is included in the repo for now
|
run: vfc_ci test -g -r
|
||||||
# (we'll probably want to remove "./" if the script ends up being integrated
|
|
||||||
# in Verificarlo and becomes available system-wide)
|
|
||||||
run: ./vfc_ci test -g -r
|
|
||||||
|
|
||||||
- name: Commit test results
|
- name: Commit test results
|
||||||
run: |
|
run: |
|
||||||
git_hash=$(git rev-parse --short "$GITHUB_SHA")
|
git_hash=$(git rev-parse --short "$GITHUB_SHA")
|
||||||
|
|
||||||
git config --local user.email "action@github.com"
|
git config --local user.email "action@github.com"
|
||||||
git config --local user.name "GitHub Action"
|
git config --local user.name "GitHub Action"
|
||||||
|
|
||||||
git checkout vfc_ci_dev
|
git checkout vfc_ci_dev
|
||||||
mkdir -p vfcruns
|
mkdir -p vfcruns
|
||||||
mv *.vfcrun.h5 vfcruns
|
mv *.vfcrun.h5 vfcruns
|
||||||
git add vfcruns/*
|
git add vfcruns/*
|
||||||
git commit -m "[auto] New test results for commit ${git_hash}"
|
git commit -m "[auto] New test results for commit ${git_hash}"
|
||||||
git push
|
git push
|
||||||
|
|
||||||
- name: Upload raw results as artifacts
|
- name: Upload raw results as artifacts
|
||||||
uses: actions/upload-artifact@v2
|
uses: actions/upload-artifact@v2
|
||||||
with:
|
with:
|
||||||
|
|||||||
@ -20,7 +20,7 @@ ifeq ($(MKL),-DMKL)
|
|||||||
ifeq ($(ENV),INTEL)
|
ifeq ($(ENV),INTEL)
|
||||||
LFLAGS = -mkl=sequential # implicit
|
LFLAGS = -mkl=sequential # implicit
|
||||||
else
|
else
|
||||||
LFLAGS = $(H5LFLAGS)
|
LFLAGS = $(H5LFLAGS) -lvfc_probes
|
||||||
endif
|
endif
|
||||||
endif
|
endif
|
||||||
H5CXXFLAGS = $(CXXFLAGS)
|
H5CXXFLAGS = $(CXXFLAGS)
|
||||||
|
|||||||
28
ci/serve.py
28
ci/serve.py
@ -1,28 +0,0 @@
|
|||||||
# Server for the Verificarlo CI report. This is simply a wrapper to avoid
|
|
||||||
# calling Bokeh directly.
|
|
||||||
|
|
||||||
import os
|
|
||||||
|
|
||||||
|
|
||||||
def serve(show, git_directory, git_url, port, allow_origin, logo_url):
|
|
||||||
|
|
||||||
# Prepare arguments
|
|
||||||
show = "--show" if show else ""
|
|
||||||
|
|
||||||
git = ""
|
|
||||||
if git_directory is not None:
|
|
||||||
git = "git directory %s" % git_directory
|
|
||||||
if git_url is not None:
|
|
||||||
git = "git url %s" % git_url
|
|
||||||
|
|
||||||
logo = ""
|
|
||||||
if logo_url is not None:
|
|
||||||
logo = "logo %s" % logo_url
|
|
||||||
|
|
||||||
dirname = os.path.dirname(__file__)
|
|
||||||
|
|
||||||
# Call the "bokeh serve" command on the system
|
|
||||||
command = "bokeh serve %s/vfc_ci_report %s --allow-websocket-origin=%s:%s --port %s --args %s %s" \
|
|
||||||
% (dirname, show, allow_origin, port, port, git, logo)
|
|
||||||
|
|
||||||
os.system(command)
|
|
||||||
146
ci/setup.py
146
ci/setup.py
@ -1,146 +0,0 @@
|
|||||||
# Helper script to set up Verificarlo CI on the current branch
|
|
||||||
|
|
||||||
import git
|
|
||||||
|
|
||||||
import sys
|
|
||||||
import os
|
|
||||||
from jinja2 import Environment, FileSystemLoader
|
|
||||||
|
|
||||||
##########################################################################
|
|
||||||
|
|
||||||
# Helper functions
|
|
||||||
|
|
||||||
|
|
||||||
def gen_readme(dev_branch, ci_branch):
|
|
||||||
|
|
||||||
# Init template loader
|
|
||||||
path = os.path.dirname(os.path.abspath(__file__))
|
|
||||||
env = Environment(loader=FileSystemLoader(path))
|
|
||||||
template = env.get_template("workflow_templates/ci_README.j2.md")
|
|
||||||
|
|
||||||
# Render template
|
|
||||||
render = template.render(dev_branch=dev_branch, ci_branch=ci_branch)
|
|
||||||
|
|
||||||
# Write template
|
|
||||||
with open("README.md", "w") as fh:
|
|
||||||
fh.write(render)
|
|
||||||
|
|
||||||
|
|
||||||
def gen_workflow(git_host, dev_branch, ci_branch, repo):
|
|
||||||
|
|
||||||
# Init template loader
|
|
||||||
path = os.path.dirname(os.path.abspath(__file__))
|
|
||||||
env = Environment(loader=FileSystemLoader(path))
|
|
||||||
|
|
||||||
if git_host == "github":
|
|
||||||
# Load template
|
|
||||||
template = env.get_template(
|
|
||||||
"workflow_templates/vfc_test_workflow.j2.yml")
|
|
||||||
|
|
||||||
# Render it
|
|
||||||
render = template.render(dev_branch=dev_branch, ci_branch=ci_branch)
|
|
||||||
|
|
||||||
# Write the file
|
|
||||||
filename = ".github/workflows/vfc_test_workflow.yml"
|
|
||||||
os.makedirs(os.path.dirname(filename), exist_ok=True)
|
|
||||||
with open(filename, "w") as fh:
|
|
||||||
fh.write(render)
|
|
||||||
|
|
||||||
if git_host == "gitlab":
|
|
||||||
template = env.get_template("workflow_templates/gitlab-ci.j2.yml")
|
|
||||||
|
|
||||||
# Ask for the user who will run the jobs (Gitlab specific)
|
|
||||||
username = input(
|
|
||||||
"[vfc_ci] Enter the name of the user who will run the CI jobs:")
|
|
||||||
email = input(
|
|
||||||
"[vfc_ci] Enter the e-mail of the user who will run the CI jobs:")
|
|
||||||
|
|
||||||
remote_url = repo.remotes[0].config_reader.get("url")
|
|
||||||
remote_url = remote_url.replace("http://", "")
|
|
||||||
remote_url = remote_url.replace("https://", "")
|
|
||||||
|
|
||||||
render = template.render(
|
|
||||||
dev_branch=dev_branch,
|
|
||||||
ci_branch=ci_branch,
|
|
||||||
username=username,
|
|
||||||
email=email,
|
|
||||||
remote_url=remote_url
|
|
||||||
)
|
|
||||||
|
|
||||||
filename = ".gitlab-ci.yml"
|
|
||||||
with open(filename, "w") as fh:
|
|
||||||
fh.write(render)
|
|
||||||
|
|
||||||
|
|
||||||
##########################################################################
|
|
||||||
|
|
||||||
def setup(git_host):
|
|
||||||
|
|
||||||
# Init repo and make sure that the workflow setup is possible
|
|
||||||
|
|
||||||
repo = git.Repo(".")
|
|
||||||
repo.remotes.origin.fetch()
|
|
||||||
|
|
||||||
# Make sure that repository is clean
|
|
||||||
assert(not repo.is_dirty()), "Error [vfc_ci]: Unstaged changes detected " \
|
|
||||||
"in your work tree."
|
|
||||||
|
|
||||||
dev_branch = repo.active_branch
|
|
||||||
dev_branch_name = str(dev_branch)
|
|
||||||
dev_remote = dev_branch.tracking_branch()
|
|
||||||
|
|
||||||
# Make sure that the active branch (on which to setup the workflow) has a
|
|
||||||
# remote
|
|
||||||
assert(dev_remote is not None), "Error [vfc_ci]: The current branch doesn't " \
|
|
||||||
"have a remote."
|
|
||||||
|
|
||||||
# Make sure that we are not behind the remote (so we can push safely later)
|
|
||||||
rev = "%s...%s" % (dev_branch_name, str(dev_remote))
|
|
||||||
commits_behind = list(repo.iter_commits(rev))
|
|
||||||
assert(commits_behind == []), "Error [vfc_ci]: The local branch seems " \
|
|
||||||
"to be at least one commit behind remote."
|
|
||||||
|
|
||||||
# Commit the workflow on the current (dev) branch
|
|
||||||
|
|
||||||
ci_branch_name = "vfc_ci_%s" % dev_branch_name
|
|
||||||
gen_workflow(git_host, dev_branch_name, ci_branch_name, repo)
|
|
||||||
repo.git.add(".")
|
|
||||||
repo.index.commit("[auto] Set up Verificarlo CI on this branch")
|
|
||||||
repo.remote(name="origin").push()
|
|
||||||
|
|
||||||
# Create the CI branch (orphan branch with a readme on it)
|
|
||||||
# (see : https://github.com/gitpython-developers/GitPython/issues/615)
|
|
||||||
|
|
||||||
repo.head.reference = git.Head(repo, "refs/heads/" + ci_branch_name)
|
|
||||||
|
|
||||||
repo.index.remove(["*"])
|
|
||||||
gen_readme(dev_branch_name, ci_branch_name)
|
|
||||||
repo.index.add(["README.md"])
|
|
||||||
|
|
||||||
repo.index.commit(
|
|
||||||
"[auto] Create the Verificarlo CI branch for %s" % dev_branch_name,
|
|
||||||
parent_commits=None
|
|
||||||
)
|
|
||||||
repo.remote(name="origin").push(
|
|
||||||
refspec="%s:%s" % (ci_branch_name, ci_branch_name)
|
|
||||||
)
|
|
||||||
|
|
||||||
# Force checkout back to the original (dev) branch
|
|
||||||
repo.git.checkout(dev_branch_name, force=True)
|
|
||||||
|
|
||||||
# Print termination messages
|
|
||||||
|
|
||||||
print(
|
|
||||||
"Info [vfc_ci]: A Verificarlo CI workflow has been setup on "
|
|
||||||
"%s." % dev_branch_name
|
|
||||||
)
|
|
||||||
print(
|
|
||||||
"Info [vfc_ci]: Make sure that you have a \"vfc_tests_config.json\" on "
|
|
||||||
"this branch. You can also perform a \"vfc_ci test\" dry run before "
|
|
||||||
"pushing other commits.")
|
|
||||||
|
|
||||||
if git_host == "gitlab":
|
|
||||||
print(
|
|
||||||
"Info [vfc_ci]: Since you are using GitLab, make sure that you "
|
|
||||||
"have created an access token for the user you specified (registered "
|
|
||||||
"as a variable called \"CI_PUSH_TOKEN\" in your repository).")
|
|
||||||
367
ci/test.py
367
ci/test.py
@ -1,367 +0,0 @@
|
|||||||
# This script reads the vfc_tests_config.json file and executes tests accordingly
|
|
||||||
# It will also generate a ... .vfcrunh5 file with the results of the run
|
|
||||||
|
|
||||||
import sigdigits as sd
|
|
||||||
import scipy.stats
|
|
||||||
import numpy as np
|
|
||||||
import pandas as pd
|
|
||||||
import os
|
|
||||||
import json
|
|
||||||
|
|
||||||
import calendar
|
|
||||||
import time
|
|
||||||
|
|
||||||
# Forcing an older pickle protocol allows backwards compatibility when reading
|
|
||||||
# HDF5 written in 3.8+ using an older version of Python
|
|
||||||
import pickle
|
|
||||||
pickle.HIGHEST_PROTOCOL = 4
|
|
||||||
|
|
||||||
|
|
||||||
# Magic numbers
|
|
||||||
min_pvalue = 0.05
|
|
||||||
max_zscore = 3
|
|
||||||
|
|
||||||
|
|
||||||
##########################################################################
|
|
||||||
|
|
||||||
# Helper functions
|
|
||||||
|
|
||||||
# Read a CSV file outputted by vfc_probe as a Pandas dataframe
|
|
||||||
def read_probes_csv(filepath, backend, warnings, execution_data):
|
|
||||||
|
|
||||||
try:
|
|
||||||
results = pd.read_csv(filepath)
|
|
||||||
|
|
||||||
except FileNotFoundError:
|
|
||||||
print(
|
|
||||||
"Warning [vfc_ci]: Probes not found, your code might have crashed "
|
|
||||||
"or you might have forgotten to call vfc_dump_probes"
|
|
||||||
)
|
|
||||||
warnings.append(execution_data)
|
|
||||||
return pd.DataFrame(
|
|
||||||
columns=["test", "variable", "values", "vfc_backend"]
|
|
||||||
)
|
|
||||||
|
|
||||||
except Exception:
|
|
||||||
print(
|
|
||||||
"Warning [vfc_ci]: Your probes could not be read for some unknown "
|
|
||||||
"reason"
|
|
||||||
)
|
|
||||||
warnings.append(execution_data)
|
|
||||||
return pd.DataFrame(
|
|
||||||
columns=["test", "variable", "values", "vfc_backend"]
|
|
||||||
)
|
|
||||||
|
|
||||||
if len(results) == 0:
|
|
||||||
print(
|
|
||||||
"Warning [vfc_ci]: Probes empty, it looks like you have dumped "
|
|
||||||
"them without calling vfc_put_probe"
|
|
||||||
)
|
|
||||||
warnings.append(execution_data)
|
|
||||||
|
|
||||||
# Once the CSV has been opened and validated, return its content
|
|
||||||
results["value"] = results["value"].apply(lambda x: float.fromhex(x))
|
|
||||||
results.rename(columns={"value": "values"}, inplace=True)
|
|
||||||
|
|
||||||
results["vfc_backend"] = backend
|
|
||||||
|
|
||||||
return results
|
|
||||||
|
|
||||||
|
|
||||||
# Wrappers to sd.significant_digits (returns results in base 2)
|
|
||||||
|
|
||||||
def significant_digits(x):
|
|
||||||
|
|
||||||
# If the null hypothesis is rejected, call sigdigits with the General
|
|
||||||
# formula:
|
|
||||||
if x.pvalue < min_pvalue:
|
|
||||||
# In a pandas DF, "values" actually refers to the array of columns, and
|
|
||||||
# not the column named "values"
|
|
||||||
distribution = x.values[3]
|
|
||||||
distribution = distribution.reshape(len(distribution), 1)
|
|
||||||
|
|
||||||
# The distribution's empirical average will be used as the reference
|
|
||||||
mu = np.array([x.mu])
|
|
||||||
|
|
||||||
s = sd.significant_digits(
|
|
||||||
distribution,
|
|
||||||
mu,
|
|
||||||
precision=sd.Precision.Relative,
|
|
||||||
method=sd.Method.General,
|
|
||||||
|
|
||||||
probability=0.9,
|
|
||||||
confidence=0.95
|
|
||||||
)
|
|
||||||
|
|
||||||
# s is returned inside a list
|
|
||||||
return s[0]
|
|
||||||
|
|
||||||
# Else, manually compute sMCA (Stott-Parker formula)
|
|
||||||
else:
|
|
||||||
return -np.log2(np.absolute(x.sigma / x.mu))
|
|
||||||
|
|
||||||
|
|
||||||
def significant_digits_lower_bound(x):
|
|
||||||
# If the null hypothesis is rejected, no lower bound
|
|
||||||
if x.pvalue < min_pvalue:
|
|
||||||
return x.s2
|
|
||||||
|
|
||||||
# Else, the lower bound will be computed with p= .9 alpha-1=.95
|
|
||||||
else:
|
|
||||||
distribution = x.values[3]
|
|
||||||
distribution = distribution.reshape(len(distribution), 1)
|
|
||||||
|
|
||||||
mu = np.array([x.mu])
|
|
||||||
|
|
||||||
s = sd.significant_digits(
|
|
||||||
distribution,
|
|
||||||
mu,
|
|
||||||
precision=sd.Precision.Relative,
|
|
||||||
method=sd.Method.CNH,
|
|
||||||
|
|
||||||
probability=0.9,
|
|
||||||
confidence=0.95
|
|
||||||
)
|
|
||||||
|
|
||||||
return s[0]
|
|
||||||
|
|
||||||
|
|
||||||
##########################################################################
|
|
||||||
|
|
||||||
# Main functions
|
|
||||||
|
|
||||||
|
|
||||||
# Open and read the tests config file
|
|
||||||
def read_config():
|
|
||||||
try:
|
|
||||||
with open("vfc_tests_config.json", "r") as file:
|
|
||||||
data = file.read()
|
|
||||||
|
|
||||||
except FileNotFoundError as e:
|
|
||||||
e.strerror = "Error [vfc_ci]: This file is required to describe the tests "\
|
|
||||||
"to run and generate a Verificarlo run file"
|
|
||||||
raise e
|
|
||||||
|
|
||||||
return json.loads(data)
|
|
||||||
|
|
||||||
|
|
||||||
# Set up metadata
|
|
||||||
def generate_metadata(is_git_commit):
|
|
||||||
|
|
||||||
# Metadata and filename are initiated as if no commit was associated
|
|
||||||
metadata = {
|
|
||||||
"timestamp": calendar.timegm(time.gmtime()),
|
|
||||||
"is_git_commit": is_git_commit,
|
|
||||||
"hash": "",
|
|
||||||
"author": "",
|
|
||||||
"message": ""
|
|
||||||
}
|
|
||||||
|
|
||||||
if is_git_commit:
|
|
||||||
print("Fetching metadata from last commit...")
|
|
||||||
from git import Repo
|
|
||||||
|
|
||||||
repo = Repo(".")
|
|
||||||
head_commit = repo.head.commit
|
|
||||||
|
|
||||||
metadata["timestamp"] = head_commit.authored_date
|
|
||||||
|
|
||||||
metadata["hash"] = str(head_commit)[0:7]
|
|
||||||
metadata["author"] = "%s <%s>" \
|
|
||||||
% (str(head_commit.author), head_commit.author.email)
|
|
||||||
metadata["message"] = head_commit.message.split("\n")[0]
|
|
||||||
|
|
||||||
return metadata
|
|
||||||
|
|
||||||
|
|
||||||
# Execute tests and collect results in a Pandas dataframe (+ dataprocessing)
|
|
||||||
def run_tests(config):
|
|
||||||
|
|
||||||
# Run the build command
|
|
||||||
print("Info [vfc_ci]: Building tests...")
|
|
||||||
os.system(config["make_command"])
|
|
||||||
|
|
||||||
# This is an array of Pandas dataframes for now
|
|
||||||
data = []
|
|
||||||
|
|
||||||
# Create tmp folder to export results
|
|
||||||
os.system("mkdir .vfcruns.tmp")
|
|
||||||
n_files = 0
|
|
||||||
|
|
||||||
# This will contain all executables/repetition numbers from which we could
|
|
||||||
# not get any data
|
|
||||||
warnings = []
|
|
||||||
|
|
||||||
# Tests execution loop
|
|
||||||
for executable in config["executables"]:
|
|
||||||
print("Info [vfc_ci]: Running executable :",
|
|
||||||
executable["executable"], "...")
|
|
||||||
|
|
||||||
parameters = ""
|
|
||||||
if "parameters" in executable:
|
|
||||||
parameters = executable["parameters"]
|
|
||||||
|
|
||||||
for backend in executable["vfc_backends"]:
|
|
||||||
|
|
||||||
export_backend = "VFC_BACKENDS=\"" + backend["name"] + "\" "
|
|
||||||
command = "./" + executable["executable"] + " " + parameters
|
|
||||||
|
|
||||||
repetitions = 1
|
|
||||||
if "repetitions" in backend:
|
|
||||||
repetitions = backend["repetitions"]
|
|
||||||
|
|
||||||
# Run test repetitions and save results
|
|
||||||
for i in range(repetitions):
|
|
||||||
file = ".vfcruns.tmp/%s.csv" % str(n_files)
|
|
||||||
export_output = "VFC_PROBES_OUTPUT=\"%s\" " % file
|
|
||||||
os.system(export_output + export_backend + command)
|
|
||||||
|
|
||||||
# This will only be used if we need to append this exec to the
|
|
||||||
# warnings list
|
|
||||||
execution_data = {
|
|
||||||
"executable": executable["executable"],
|
|
||||||
"backend": backend["name"],
|
|
||||||
"repetition": i + 1
|
|
||||||
}
|
|
||||||
|
|
||||||
data.append(read_probes_csv(
|
|
||||||
file,
|
|
||||||
backend["name"],
|
|
||||||
warnings,
|
|
||||||
execution_data
|
|
||||||
))
|
|
||||||
|
|
||||||
n_files = n_files + 1
|
|
||||||
|
|
||||||
# Clean CSV output files (by deleting the tmp folder)
|
|
||||||
os.system("rm -rf .vfcruns.tmp")
|
|
||||||
|
|
||||||
# Combine all separate executions in one dataframe
|
|
||||||
data = pd.concat(data, sort=False, ignore_index=True)
|
|
||||||
data = data.groupby(["test", "vfc_backend", "variable"])\
|
|
||||||
.values.apply(list).reset_index()
|
|
||||||
|
|
||||||
# Make sure we have some data to work on
|
|
||||||
assert(len(data) != 0), "Error [vfc_ci]: No data have been generated " \
|
|
||||||
"by your tests executions, aborting run without writing results file"
|
|
||||||
|
|
||||||
return data, warnings
|
|
||||||
|
|
||||||
# Data processing
|
|
||||||
|
|
||||||
|
|
||||||
def data_processing(data):
|
|
||||||
|
|
||||||
data["values"] = data["values"].apply(lambda x: np.array(x).astype(float))
|
|
||||||
|
|
||||||
# Get empirical average, standard deviation and p-value
|
|
||||||
data["mu"] = data["values"].apply(np.average)
|
|
||||||
data["sigma"] = data["values"].apply(np.std)
|
|
||||||
data["pvalue"] = data["values"].apply(
|
|
||||||
lambda x: scipy.stats.shapiro(x).pvalue)
|
|
||||||
|
|
||||||
# Significant digits
|
|
||||||
data["s2"] = data.apply(significant_digits, axis=1)
|
|
||||||
data["s10"] = data["s2"].apply(lambda x: sd.change_base(x, 10))
|
|
||||||
|
|
||||||
# Lower bound of the confidence interval using the sigdigits module
|
|
||||||
data["s2_lower_bound"] = data.apply(significant_digits_lower_bound, axis=1)
|
|
||||||
data["s10_lower_bound"] = data["s2_lower_bound"].apply(
|
|
||||||
lambda x: sd.change_base(x, 10))
|
|
||||||
|
|
||||||
# Compute moments of the distribution
|
|
||||||
# (including a new distribution obtained by filtering outliers)
|
|
||||||
data["values"] = data["values"].apply(np.sort)
|
|
||||||
|
|
||||||
data["mu"] = data["values"].apply(np.average)
|
|
||||||
data["min"] = data["values"].apply(np.min)
|
|
||||||
data["quantile25"] = data["values"].apply(np.quantile, args=(0.25,))
|
|
||||||
data["quantile50"] = data["values"].apply(np.quantile, args=(0.50,))
|
|
||||||
data["quantile75"] = data["values"].apply(np.quantile, args=(0.75,))
|
|
||||||
data["max"] = data["values"].apply(np.max)
|
|
||||||
|
|
||||||
data["nsamples"] = data["values"].apply(len)
|
|
||||||
|
|
||||||
# Display all executions that resulted in a warning
|
|
||||||
|
|
||||||
|
|
||||||
def show_warnings(warnings):
|
|
||||||
if len(warnings) > 0:
|
|
||||||
print(
|
|
||||||
"Warning [vfc_ci]: Some of your runs could not generate any data "
|
|
||||||
"(for instance because your code crashed) and resulted in "
|
|
||||||
"warnings. Here is the complete list :"
|
|
||||||
)
|
|
||||||
|
|
||||||
for i in range(0, len(warnings)):
|
|
||||||
print("- Warning %s:" % i)
|
|
||||||
|
|
||||||
print(" Executable: %s" % warnings[i]["executable"])
|
|
||||||
print(" Backend: %s" % warnings[i]["backend"])
|
|
||||||
print(" Repetition: %s" % warnings[i]["repetition"])
|
|
||||||
|
|
||||||
|
|
||||||
##########################################################################
|
|
||||||
|
|
||||||
# Entry point
|
|
||||||
|
|
||||||
def run(is_git_commit, export_raw_values, dry_run):
|
|
||||||
|
|
||||||
# Get config, metadata and data
|
|
||||||
print("Info [vfc_ci]: Reading tests config file...")
|
|
||||||
config = read_config()
|
|
||||||
|
|
||||||
print("Info [vfc_ci]: Generating run metadata...")
|
|
||||||
metadata = generate_metadata(is_git_commit)
|
|
||||||
|
|
||||||
data, warnings = run_tests(config)
|
|
||||||
show_warnings(warnings)
|
|
||||||
|
|
||||||
# Data processing
|
|
||||||
print("Info [vfc_ci]: Processing data...")
|
|
||||||
data_processing(data)
|
|
||||||
|
|
||||||
# Prepare data for export (by creating a proper index and linking run
|
|
||||||
# timestamp)
|
|
||||||
data = data.set_index(["test", "variable", "vfc_backend"]).sort_index()
|
|
||||||
data["timestamp"] = metadata["timestamp"]
|
|
||||||
|
|
||||||
filename = metadata["hash"] if is_git_commit else str(
|
|
||||||
metadata["timestamp"])
|
|
||||||
|
|
||||||
# Prepare metadata for export
|
|
||||||
metadata = pd.DataFrame.from_dict([metadata])
|
|
||||||
metadata = metadata.set_index("timestamp")
|
|
||||||
|
|
||||||
# NOTE : Exporting to HDF5 requires to install "tables" on the system
|
|
||||||
|
|
||||||
# Export raw data if needed
|
|
||||||
if export_raw_values and not dry_run:
|
|
||||||
data.to_hdf(filename + ".vfcraw.h5", key="data")
|
|
||||||
metadata.to_hdf(filename + ".vfcraw.h5", key="metadata")
|
|
||||||
|
|
||||||
# Export data
|
|
||||||
del data["values"]
|
|
||||||
if not dry_run:
|
|
||||||
data.to_hdf(filename + ".vfcrun.h5", key="data")
|
|
||||||
metadata.to_hdf(filename + ".vfcrun.h5", key="metadata")
|
|
||||||
|
|
||||||
# Print termination messages
|
|
||||||
print(
|
|
||||||
"Info [vfc_ci]: The results have been successfully written to "
|
|
||||||
"%s.vfcrun.h5."
|
|
||||||
% filename
|
|
||||||
)
|
|
||||||
|
|
||||||
if export_raw_values:
|
|
||||||
print(
|
|
||||||
"Info [vfc_ci]: A file containing the raw values has also been "
|
|
||||||
"created : %s.vfcraw.h5."
|
|
||||||
% filename
|
|
||||||
)
|
|
||||||
|
|
||||||
if dry_run:
|
|
||||||
print(
|
|
||||||
"Info [vfc_ci]: The dry run flag was enabled, so no files were "
|
|
||||||
"actually created."
|
|
||||||
)
|
|
||||||
@ -1,542 +0,0 @@
|
|||||||
# Manage the view comparing a variable over different runs
|
|
||||||
|
|
||||||
import time
|
|
||||||
|
|
||||||
import pandas as pd
|
|
||||||
|
|
||||||
from math import pi
|
|
||||||
|
|
||||||
from bokeh.plotting import figure, curdoc
|
|
||||||
from bokeh.embed import components
|
|
||||||
from bokeh.models import Select, ColumnDataSource, Panel, Tabs, HoverTool, \
|
|
||||||
TextInput, CheckboxGroup, TapTool, CustomJS
|
|
||||||
|
|
||||||
import helper
|
|
||||||
import plot
|
|
||||||
|
|
||||||
|
|
||||||
##########################################################################
|
|
||||||
|
|
||||||
|
|
||||||
class CompareRuns:
|
|
||||||
|
|
||||||
# Helper functions related to CompareRuns
|
|
||||||
|
|
||||||
# From an array of timestamps, returns the array of runs names (for the x
|
|
||||||
# axis ticks), as well as the metadata (in a dict of arrays) associated to
|
|
||||||
# this array (for the tooltips)
|
|
||||||
def gen_x_series(self, timestamps):
|
|
||||||
|
|
||||||
# Initialize the objects to return
|
|
||||||
x_series = []
|
|
||||||
x_metadata = dict(
|
|
||||||
date=[],
|
|
||||||
is_git_commit=[],
|
|
||||||
hash=[],
|
|
||||||
author=[],
|
|
||||||
message=[]
|
|
||||||
)
|
|
||||||
|
|
||||||
# n == 0 means we want all runs, we also make sure not to go out of
|
|
||||||
# bound if asked for more runs than we have
|
|
||||||
n = self.current_n_runs
|
|
||||||
if n == 0 or n > len(timestamps):
|
|
||||||
n = len(timestamps)
|
|
||||||
|
|
||||||
for i in range(0, n):
|
|
||||||
# Get metadata associated to this run
|
|
||||||
row_metadata = helper.get_metadata(
|
|
||||||
self.metadata, timestamps[-i - 1])
|
|
||||||
date = time.ctime(timestamps[-i - 1])
|
|
||||||
|
|
||||||
# Fill the x series
|
|
||||||
str = row_metadata["name"]
|
|
||||||
x_series.insert(0, helper.get_metadata(
|
|
||||||
self.metadata, timestamps[-i - 1])["name"])
|
|
||||||
|
|
||||||
# Fill the metadata lists
|
|
||||||
x_metadata["date"].insert(0, date)
|
|
||||||
x_metadata["is_git_commit"].insert(
|
|
||||||
0, row_metadata["is_git_commit"])
|
|
||||||
x_metadata["hash"].insert(0, row_metadata["hash"])
|
|
||||||
x_metadata["author"].insert(0, row_metadata["author"])
|
|
||||||
x_metadata["message"].insert(0, row_metadata["message"])
|
|
||||||
|
|
||||||
return x_series, x_metadata
|
|
||||||
|
|
||||||
# Plots update function
|
|
||||||
|
|
||||||
def update_plots(self):
|
|
||||||
|
|
||||||
# Select all data matching current test/var/backend
|
|
||||||
|
|
||||||
runs = self.data.loc[[self.widgets["select_test"].value],
|
|
||||||
self.widgets["select_var"].value,
|
|
||||||
self.widgets["select_backend"].value]
|
|
||||||
|
|
||||||
timestamps = runs["timestamp"]
|
|
||||||
x_series, x_metadata = self.gen_x_series(timestamps.sort_values())
|
|
||||||
|
|
||||||
# Update source
|
|
||||||
|
|
||||||
main_dict = runs.to_dict("series")
|
|
||||||
main_dict["x"] = x_series
|
|
||||||
|
|
||||||
# Add metadata (for tooltip)
|
|
||||||
main_dict.update(x_metadata)
|
|
||||||
|
|
||||||
# Select the last n runs only
|
|
||||||
n = self.current_n_runs
|
|
||||||
main_dict = {key: value[-n:] for key, value in main_dict.items()}
|
|
||||||
|
|
||||||
# Generate ColumnDataSources for the 3 dotplots
|
|
||||||
for stat in ["sigma", "s10", "s2"]:
|
|
||||||
dict = {
|
|
||||||
"%s_x" % stat: main_dict["x"],
|
|
||||||
|
|
||||||
"is_git_commit": main_dict["is_git_commit"],
|
|
||||||
"date": main_dict["date"],
|
|
||||||
"hash": main_dict["hash"],
|
|
||||||
"author": main_dict["author"],
|
|
||||||
"message": main_dict["message"],
|
|
||||||
|
|
||||||
stat: main_dict[stat],
|
|
||||||
|
|
||||||
"nsamples": main_dict["nsamples"],
|
|
||||||
}
|
|
||||||
|
|
||||||
if stat == "s10" or stat == "s2":
|
|
||||||
dict["%s_lower_bound" %
|
|
||||||
stat] = main_dict["%s_lower_bound" %
|
|
||||||
stat]
|
|
||||||
|
|
||||||
# Filter outliers if the box is checked
|
|
||||||
if len(self.widgets["outliers_filtering_compare"].active) > 0:
|
|
||||||
outliers = helper.detect_outliers(dict[stat])
|
|
||||||
dict[stat] = helper.remove_outliers(dict[stat], outliers)
|
|
||||||
dict["%s_x" % stat] = helper.remove_outliers(
|
|
||||||
dict["%s_x" % stat], outliers)
|
|
||||||
|
|
||||||
# Assign ColumnDataSource
|
|
||||||
self.sources["%s_source" % stat].data = dict
|
|
||||||
|
|
||||||
# Generate ColumnDataSource for the boxplot
|
|
||||||
dict = {
|
|
||||||
"is_git_commit": main_dict["is_git_commit"],
|
|
||||||
"date": main_dict["date"],
|
|
||||||
"hash": main_dict["hash"],
|
|
||||||
"author": main_dict["author"],
|
|
||||||
"message": main_dict["message"],
|
|
||||||
|
|
||||||
"x": main_dict["x"],
|
|
||||||
"min": main_dict["min"],
|
|
||||||
"quantile25": main_dict["quantile25"],
|
|
||||||
"quantile50": main_dict["quantile50"],
|
|
||||||
"quantile75": main_dict["quantile75"],
|
|
||||||
"max": main_dict["max"],
|
|
||||||
"mu": main_dict["mu"],
|
|
||||||
"pvalue": main_dict["pvalue"],
|
|
||||||
|
|
||||||
"nsamples": main_dict["nsamples"]
|
|
||||||
}
|
|
||||||
|
|
||||||
self.sources["boxplot_source"].data = dict
|
|
||||||
|
|
||||||
# Update x axis
|
|
||||||
|
|
||||||
helper.reset_x_range(
|
|
||||||
self.plots["boxplot"],
|
|
||||||
self.sources["boxplot_source"].data["x"]
|
|
||||||
)
|
|
||||||
helper.reset_x_range(
|
|
||||||
self.plots["sigma_plot"],
|
|
||||||
self.sources["sigma_source"].data["sigma_x"]
|
|
||||||
)
|
|
||||||
helper.reset_x_range(
|
|
||||||
self.plots["s10_plot"],
|
|
||||||
self.sources["s10_source"].data["s10_x"]
|
|
||||||
)
|
|
||||||
helper.reset_x_range(
|
|
||||||
self.plots["s2_plot"],
|
|
||||||
self.sources["s2_source"].data["s2_x"]
|
|
||||||
)
|
|
||||||
|
|
||||||
# Widgets' callback functions
|
|
||||||
|
|
||||||
def update_test(self, attrname, old, new):
|
|
||||||
|
|
||||||
# If the value is updated by the CustomJS, self.widgets["select_var"].value
|
|
||||||
# won't be updated, so we have to look for that case and assign it
|
|
||||||
# manually
|
|
||||||
|
|
||||||
# "new" should be a list when updated by CustomJS
|
|
||||||
if isinstance(new, list):
|
|
||||||
# If filtering removed all options, we might have an empty list
|
|
||||||
# (in this case, we just skip the callback and do nothing)
|
|
||||||
if len(new) > 0:
|
|
||||||
new = new[0]
|
|
||||||
else:
|
|
||||||
return
|
|
||||||
|
|
||||||
if new != self.widgets["select_test"].value:
|
|
||||||
# The callback will be triggered again with the updated value
|
|
||||||
self.widgets["select_test"].value = new
|
|
||||||
return
|
|
||||||
|
|
||||||
# New list of available vars
|
|
||||||
self.vars = self.data.loc[new]\
|
|
||||||
.index.get_level_values("variable").drop_duplicates().tolist()
|
|
||||||
self.widgets["select_var"].options = self.vars
|
|
||||||
|
|
||||||
# Reset var selection if old one is not available in new vars
|
|
||||||
if self.widgets["select_var"].value not in self.vars:
|
|
||||||
self.widgets["select_var"].value = self.vars[0]
|
|
||||||
# The update_var callback will be triggered by the assignment
|
|
||||||
|
|
||||||
else:
|
|
||||||
# Trigger the callback manually (since the plots need to be updated
|
|
||||||
# anyway)
|
|
||||||
self.update_var("", "", self.widgets["select_var"].value)
|
|
||||||
|
|
||||||
def update_var(self, attrname, old, new):
|
|
||||||
|
|
||||||
# If the value is updated by the CustomJS, self.widgets["select_var"].value
|
|
||||||
# won't be updated, so we have to look for that case and assign it
|
|
||||||
# manually
|
|
||||||
|
|
||||||
# new should be a list when updated by CustomJS
|
|
||||||
if isinstance(new, list):
|
|
||||||
new = new[0]
|
|
||||||
|
|
||||||
if new != self.widgets["select_var"].value:
|
|
||||||
# The callback will be triggered again with the updated value
|
|
||||||
self.widgets["select_var"].value = new
|
|
||||||
return
|
|
||||||
|
|
||||||
# New list of available backends
|
|
||||||
self.backends = self.data.loc[self.widgets["select_test"].value, self.widgets["select_var"].value]\
|
|
||||||
.index.get_level_values("vfc_backend").drop_duplicates().tolist()
|
|
||||||
self.widgets["select_backend"].options = self.backends
|
|
||||||
|
|
||||||
# Reset backend selection if old one is not available in new backends
|
|
||||||
if self.widgets["select_backend"].value not in self.backends:
|
|
||||||
self.widgets["select_backend"].value = self.backends[0]
|
|
||||||
# The update_backend callback will be triggered by the assignment
|
|
||||||
|
|
||||||
else:
|
|
||||||
# Trigger the callback manually (since the plots need to be updated
|
|
||||||
# anyway)
|
|
||||||
self.update_backend("", "", self.widgets["select_backend"].value)
|
|
||||||
|
|
||||||
def update_backend(self, attrname, old, new):
|
|
||||||
|
|
||||||
# Simply update plots, since no other data is affected
|
|
||||||
self.update_plots()
|
|
||||||
|
|
||||||
def update_n_runs(self, attrname, old, new):
|
|
||||||
# Simply update runs selection (value and string display)
|
|
||||||
self.widgets["select_n_runs"].value = new
|
|
||||||
self.current_n_runs = self.n_runs_dict[self.widgets["select_n_runs"].value]
|
|
||||||
|
|
||||||
self.update_plots()
|
|
||||||
|
|
||||||
def update_outliers_filtering(self, attrname, old, new):
|
|
||||||
self.update_plots()
|
|
||||||
|
|
||||||
# Bokeh setup functions
|
|
||||||
|
|
||||||
def setup_plots(self):
|
|
||||||
|
|
||||||
tools = "pan, wheel_zoom, xwheel_zoom, ywheel_zoom, reset, save"
|
|
||||||
|
|
||||||
# Custom JS callback that will be used when tapping on a run
|
|
||||||
# Only switches the view, a server callback is required to update plots
|
|
||||||
# (defined inside template to avoid bloating server w/ too much JS code)
|
|
||||||
js_tap_callback = "goToInspectRuns();"
|
|
||||||
|
|
||||||
# Box plot
|
|
||||||
self.plots["boxplot"] = figure(
|
|
||||||
name="boxplot", title="Variable distribution over runs",
|
|
||||||
plot_width=900, plot_height=400, x_range=[""],
|
|
||||||
tools=tools, sizing_mode="scale_width"
|
|
||||||
)
|
|
||||||
|
|
||||||
box_tooltips = [
|
|
||||||
("Git commit", "@is_git_commit"),
|
|
||||||
("Date", "@date"),
|
|
||||||
("Hash", "@hash"),
|
|
||||||
("Author", "@author"),
|
|
||||||
("Message", "@message"),
|
|
||||||
("Min", "@min{%0.18e}"),
|
|
||||||
("Max", "@max{%0.18e}"),
|
|
||||||
("1st quartile", "@quantile25{%0.18e}"),
|
|
||||||
("Median", "@quantile50{%0.18e}"),
|
|
||||||
("3rd quartile", "@quantile75{%0.18e}"),
|
|
||||||
("μ", "@mu{%0.18e}"),
|
|
||||||
("p-value", "@pvalue"),
|
|
||||||
("Number of samples", "@nsamples")
|
|
||||||
]
|
|
||||||
box_tooltips_formatters = {
|
|
||||||
"@min": "printf",
|
|
||||||
"@max": "printf",
|
|
||||||
"@quantile25": "printf",
|
|
||||||
"@quantile50": "printf",
|
|
||||||
"@quantile75": "printf",
|
|
||||||
"@mu": "printf"
|
|
||||||
}
|
|
||||||
|
|
||||||
plot.fill_boxplot(
|
|
||||||
self.plots["boxplot"], self.sources["boxplot_source"],
|
|
||||||
tooltips=box_tooltips,
|
|
||||||
tooltips_formatters=box_tooltips_formatters,
|
|
||||||
js_tap_callback=js_tap_callback,
|
|
||||||
server_tap_callback=self.inspect_run_callback_boxplot,
|
|
||||||
)
|
|
||||||
self.doc.add_root(self.plots["boxplot"])
|
|
||||||
|
|
||||||
# Sigma plot (bar plot)
|
|
||||||
self.plots["sigma_plot"] = figure(
|
|
||||||
name="sigma_plot", title="Standard deviation σ over runs",
|
|
||||||
plot_width=900, plot_height=400, x_range=[""],
|
|
||||||
tools=tools, sizing_mode="scale_width"
|
|
||||||
)
|
|
||||||
|
|
||||||
sigma_tooltips = [
|
|
||||||
("Git commit", "@is_git_commit"),
|
|
||||||
("Date", "@date"),
|
|
||||||
("Hash", "@hash"),
|
|
||||||
("Author", "@author"),
|
|
||||||
("Message", "@message"),
|
|
||||||
("σ", "@sigma"),
|
|
||||||
("Number of samples", "@nsamples")
|
|
||||||
]
|
|
||||||
|
|
||||||
plot.fill_dotplot(
|
|
||||||
self.plots["sigma_plot"], self.sources["sigma_source"], "sigma",
|
|
||||||
tooltips=sigma_tooltips,
|
|
||||||
js_tap_callback=js_tap_callback,
|
|
||||||
server_tap_callback=self.inspect_run_callback_sigma,
|
|
||||||
lines=True
|
|
||||||
)
|
|
||||||
self.doc.add_root(self.plots["sigma_plot"])
|
|
||||||
|
|
||||||
# s plot (bar plot with 2 tabs)
|
|
||||||
self.plots["s10_plot"] = figure(
|
|
||||||
name="s10_plot", title="Significant digits s over runs",
|
|
||||||
plot_width=900, plot_height=400, x_range=[""],
|
|
||||||
tools=tools, sizing_mode="scale_width"
|
|
||||||
)
|
|
||||||
|
|
||||||
s10_tooltips = [
|
|
||||||
("Git commit", "@is_git_commit"),
|
|
||||||
("Date", "@date"),
|
|
||||||
("Hash", "@hash"),
|
|
||||||
("Author", "@author"),
|
|
||||||
("Message", "@message"),
|
|
||||||
("s", "@s10"),
|
|
||||||
("s lower bound", "@s10_lower_bound"),
|
|
||||||
("Number of samples", "@nsamples")
|
|
||||||
]
|
|
||||||
|
|
||||||
plot.fill_dotplot(
|
|
||||||
self.plots["s10_plot"], self.sources["s10_source"], "s10",
|
|
||||||
tooltips=s10_tooltips,
|
|
||||||
js_tap_callback=js_tap_callback,
|
|
||||||
server_tap_callback=self.inspect_run_callback_s10,
|
|
||||||
lines=True,
|
|
||||||
lower_bound=True
|
|
||||||
)
|
|
||||||
s10_tab = Panel(child=self.plots["s10_plot"], title="Base 10")
|
|
||||||
|
|
||||||
self.plots["s2_plot"] = figure(
|
|
||||||
name="s2_plot", title="Significant digits s over runs",
|
|
||||||
plot_width=900, plot_height=400, x_range=[""],
|
|
||||||
tools=tools, sizing_mode="scale_width"
|
|
||||||
)
|
|
||||||
|
|
||||||
s2_tooltips = [
|
|
||||||
("Git commit", "@is_git_commit"),
|
|
||||||
("Date", "@date"),
|
|
||||||
("Hash", "@hash"),
|
|
||||||
("Author", "@author"),
|
|
||||||
("Message", "@message"),
|
|
||||||
("s", "@s2"),
|
|
||||||
("s lower bound", "@s2_lower_bound"),
|
|
||||||
("Number of samples", "@nsamples")
|
|
||||||
]
|
|
||||||
|
|
||||||
plot.fill_dotplot(
|
|
||||||
self.plots["s2_plot"], self.sources["s2_source"], "s2",
|
|
||||||
tooltips=s2_tooltips,
|
|
||||||
js_tap_callback=js_tap_callback,
|
|
||||||
server_tap_callback=self.inspect_run_callback_s2,
|
|
||||||
lines=True,
|
|
||||||
lower_bound=True
|
|
||||||
)
|
|
||||||
s2_tab = Panel(child=self.plots["s2_plot"], title="Base 2")
|
|
||||||
|
|
||||||
s_tabs = Tabs(
|
|
||||||
name="s_tabs",
|
|
||||||
tabs=[s10_tab, s2_tab],
|
|
||||||
tabs_location="below"
|
|
||||||
)
|
|
||||||
|
|
||||||
self.doc.add_root(s_tabs)
|
|
||||||
|
|
||||||
def setup_widgets(self):
|
|
||||||
|
|
||||||
# Initial selections
|
|
||||||
|
|
||||||
# Test/var/backend combination (we select all first elements at init)
|
|
||||||
self.tests = self.data\
|
|
||||||
.index.get_level_values("test").drop_duplicates().tolist()
|
|
||||||
|
|
||||||
self.vars = self.data.loc[self.tests[0]]\
|
|
||||||
.index.get_level_values("variable").drop_duplicates().tolist()
|
|
||||||
|
|
||||||
self.backends = self.data.loc[self.tests[0], self.vars[0]]\
|
|
||||||
.index.get_level_values("vfc_backend").drop_duplicates().tolist()
|
|
||||||
|
|
||||||
# Custom JS callback that will be used client side to filter selections
|
|
||||||
filter_callback_js = """
|
|
||||||
selector.options = options.filter(e => e.includes(cb_obj.value));
|
|
||||||
"""
|
|
||||||
|
|
||||||
# Test selector widget
|
|
||||||
|
|
||||||
# Number of runs to display
|
|
||||||
# The dict structure allows us to get int value from the display string
|
|
||||||
# in O(1)
|
|
||||||
self.n_runs_dict = {
|
|
||||||
"Last 3 runs": 3,
|
|
||||||
"Last 5 runs": 5,
|
|
||||||
"Last 10 runs": 10,
|
|
||||||
"All runs": 0
|
|
||||||
}
|
|
||||||
|
|
||||||
# Contains all options strings
|
|
||||||
n_runs_display = list(self.n_runs_dict.keys())
|
|
||||||
|
|
||||||
# Will be used when updating plots (contains actual number to diplay)
|
|
||||||
self.current_n_runs = self.n_runs_dict[n_runs_display[1]]
|
|
||||||
|
|
||||||
# Selector widget
|
|
||||||
self.widgets["select_test"] = Select(
|
|
||||||
name="select_test", title="Test :",
|
|
||||||
value=self.tests[0], options=self.tests
|
|
||||||
)
|
|
||||||
self.doc.add_root(self.widgets["select_test"])
|
|
||||||
self.widgets["select_test"].on_change("value", self.update_test)
|
|
||||||
self.widgets["select_test"].on_change("options", self.update_test)
|
|
||||||
|
|
||||||
# Filter widget
|
|
||||||
self.widgets["test_filter"] = TextInput(
|
|
||||||
name="test_filter", title="Tests filter:"
|
|
||||||
)
|
|
||||||
self.widgets["test_filter"].js_on_change(
|
|
||||||
"value",
|
|
||||||
CustomJS(
|
|
||||||
args=dict(
|
|
||||||
options=self.tests,
|
|
||||||
selector=self.widgets["select_test"]),
|
|
||||||
code=filter_callback_js))
|
|
||||||
self.doc.add_root(self.widgets["test_filter"])
|
|
||||||
|
|
||||||
# Number of runs to display
|
|
||||||
|
|
||||||
self.widgets["select_n_runs"] = Select(
|
|
||||||
name="select_n_runs", title="Display :",
|
|
||||||
value=n_runs_display[1], options=n_runs_display
|
|
||||||
)
|
|
||||||
self.doc.add_root(self.widgets["select_n_runs"])
|
|
||||||
self.widgets["select_n_runs"].on_change("value", self.update_n_runs)
|
|
||||||
|
|
||||||
# Variable selector widget
|
|
||||||
|
|
||||||
self.widgets["select_var"] = Select(
|
|
||||||
name="select_var", title="Variable :",
|
|
||||||
value=self.vars[0], options=self.vars
|
|
||||||
)
|
|
||||||
self.doc.add_root(self.widgets["select_var"])
|
|
||||||
self.widgets["select_var"].on_change("value", self.update_var)
|
|
||||||
self.widgets["select_var"].on_change("options", self.update_var)
|
|
||||||
|
|
||||||
# Backend selector widget
|
|
||||||
|
|
||||||
self.widgets["select_backend"] = Select(
|
|
||||||
name="select_backend", title="Verificarlo backend :",
|
|
||||||
value=self.backends[0], options=self.backends
|
|
||||||
)
|
|
||||||
self.doc.add_root(self.widgets["select_backend"])
|
|
||||||
self.widgets["select_backend"].on_change("value", self.update_backend)
|
|
||||||
|
|
||||||
# Outliers filtering checkbox
|
|
||||||
|
|
||||||
self.widgets["outliers_filtering_compare"] = CheckboxGroup(
|
|
||||||
name="outliers_filtering_compare",
|
|
||||||
labels=["Filter outliers"], active=[]
|
|
||||||
)
|
|
||||||
self.doc.add_root(self.widgets["outliers_filtering_compare"])
|
|
||||||
self.widgets["outliers_filtering_compare"]\
|
|
||||||
.on_change("active", self.update_outliers_filtering)
|
|
||||||
|
|
||||||
# Communication methods
|
|
||||||
# (to send/receive messages to/from master)
|
|
||||||
|
|
||||||
# Callback to change view of Inspect runs when data is selected
|
|
||||||
|
|
||||||
def inspect_run_callback(self, new, source_name, x_name):
|
|
||||||
|
|
||||||
# In case we just unselected everything, then do nothing
|
|
||||||
if new == []:
|
|
||||||
return
|
|
||||||
|
|
||||||
index = new[-1]
|
|
||||||
run_name = self.sources[source_name].data[x_name][index]
|
|
||||||
|
|
||||||
self.master.go_to_inspect(run_name)
|
|
||||||
|
|
||||||
# Wrappers for each plot (since new is the index of the clicked element,
|
|
||||||
# it is dependent of the plot because we could have filtered some outliers)
|
|
||||||
# There doesn't seem to be an easy way to add custom parameters to a
|
|
||||||
# Bokeh callback, so using wrappers seems to be the best solution for now
|
|
||||||
|
|
||||||
def inspect_run_callback_boxplot(self, attr, old, new):
|
|
||||||
self.inspect_run_callback(new, "boxplot_source", "x")
|
|
||||||
|
|
||||||
def inspect_run_callback_sigma(self, attr, old, new):
|
|
||||||
self.inspect_run_callback(new, "sigma_source", "sigma_x")
|
|
||||||
|
|
||||||
def inspect_run_callback_s2(self, attr, old, new):
|
|
||||||
self.inspect_run_callback(new, "s2_source", "s2_x")
|
|
||||||
|
|
||||||
def inspect_run_callback_s10(self, attr, old, new):
|
|
||||||
self.inspect_run_callback(new, "s10_source", "s10_x")
|
|
||||||
|
|
||||||
# Constructor
|
|
||||||
|
|
||||||
def __init__(self, master, doc, data, metadata):
|
|
||||||
|
|
||||||
self.master = master
|
|
||||||
|
|
||||||
self.doc = doc
|
|
||||||
self.data = data
|
|
||||||
self.metadata = metadata
|
|
||||||
|
|
||||||
self.sources = {
|
|
||||||
"boxplot_source": ColumnDataSource(data={}),
|
|
||||||
"sigma_source": ColumnDataSource(data={}),
|
|
||||||
"s10_source": ColumnDataSource(data={}),
|
|
||||||
"s2_source": ColumnDataSource(data={})
|
|
||||||
}
|
|
||||||
|
|
||||||
self.plots = {}
|
|
||||||
self.widgets = {}
|
|
||||||
|
|
||||||
# Setup Bokeh objects
|
|
||||||
self.setup_plots()
|
|
||||||
self.setup_widgets()
|
|
||||||
|
|
||||||
# At this point, everything should have been initialized, so we can
|
|
||||||
# show the plots for the first time
|
|
||||||
self.update_plots()
|
|
||||||
@ -1,170 +0,0 @@
|
|||||||
# General helper functions for both compare_runs and compare_variables
|
|
||||||
|
|
||||||
import calendar
|
|
||||||
import time
|
|
||||||
from itertools import compress
|
|
||||||
|
|
||||||
import numpy as np
|
|
||||||
|
|
||||||
# Magic numbers
|
|
||||||
max_ticks = 15
|
|
||||||
max_zscore = 3
|
|
||||||
|
|
||||||
##########################################################################
|
|
||||||
|
|
||||||
|
|
||||||
# From a timestamp, return the associated metadata as a Pandas serie
|
|
||||||
def get_metadata(metadata, timestamp):
|
|
||||||
return metadata.loc[timestamp]
|
|
||||||
|
|
||||||
|
|
||||||
# Convert a metadata Pandas series to a JS readable dict
|
|
||||||
def metadata_to_dict(metadata):
|
|
||||||
dict = metadata.to_dict()
|
|
||||||
|
|
||||||
# JS doesn't accept True for booleans, and Python doesn't accept true
|
|
||||||
# (because of the caps) => using an integer is a portable solution
|
|
||||||
dict["is_git_commit"] = 1 if dict["is_git_commit"] else 0
|
|
||||||
|
|
||||||
dict["date"] = time.ctime(metadata.name)
|
|
||||||
|
|
||||||
return dict
|
|
||||||
|
|
||||||
|
|
||||||
# Return a string that indicates the elapsed time since the run, used as the
|
|
||||||
# x-axis tick in "Compare runs" or when selecting run in "Inspect run"
|
|
||||||
def get_run_name(timestamp, hash):
|
|
||||||
|
|
||||||
gmt = time.gmtime()
|
|
||||||
now = calendar.timegm(gmt)
|
|
||||||
diff = now - timestamp
|
|
||||||
|
|
||||||
# Special case : < 1 minute (return string directly)
|
|
||||||
if diff < 60:
|
|
||||||
str = "Less than a minute ago"
|
|
||||||
|
|
||||||
if hash != "":
|
|
||||||
str = str + " (%s)" % hash
|
|
||||||
|
|
||||||
if str == get_run_name.previous:
|
|
||||||
get_run_name.counter = get_run_name.counter + 1
|
|
||||||
str = "%s (%s)" % (str, get_run_name.counter)
|
|
||||||
else:
|
|
||||||
get_run_name.counter = 0
|
|
||||||
get_run_name.previous = str
|
|
||||||
|
|
||||||
return str
|
|
||||||
|
|
||||||
# < 1 hour
|
|
||||||
if diff < 3600:
|
|
||||||
n = int(diff / 60)
|
|
||||||
str = "%s minute%s ago"
|
|
||||||
# < 1 day
|
|
||||||
elif diff < 86400:
|
|
||||||
n = int(diff / 3600)
|
|
||||||
str = "%s hour%s ago"
|
|
||||||
# < 1 week
|
|
||||||
elif diff < 604800:
|
|
||||||
n = int(diff / 86400)
|
|
||||||
str = "%s day%s ago"
|
|
||||||
# < 1 month
|
|
||||||
elif diff < 2592000:
|
|
||||||
n = int(diff / 604800)
|
|
||||||
str = "%s week%s ago"
|
|
||||||
# > 1 month
|
|
||||||
else:
|
|
||||||
n = diff / 2592000
|
|
||||||
str = "%s month%s ago"
|
|
||||||
|
|
||||||
plural = ""
|
|
||||||
if n != 1:
|
|
||||||
plural = "s"
|
|
||||||
|
|
||||||
str = str % (n, plural)
|
|
||||||
|
|
||||||
# We might want to add the git hash
|
|
||||||
if hash != "":
|
|
||||||
str = str + " (%s)" % hash
|
|
||||||
|
|
||||||
# Finally, check for duplicate with previously generated string
|
|
||||||
if str == get_run_name.previous:
|
|
||||||
# Increment the duplicate counter and add it to str
|
|
||||||
get_run_name.counter = get_run_name.counter + 1
|
|
||||||
str = "%s (%s)" % (str, get_run_name.counter)
|
|
||||||
|
|
||||||
else:
|
|
||||||
# No duplicate, reset both previously generated str and duplicate
|
|
||||||
# counter
|
|
||||||
get_run_name.counter = 0
|
|
||||||
get_run_name.previous = str
|
|
||||||
|
|
||||||
return str
|
|
||||||
|
|
||||||
|
|
||||||
# These external variables will store data about the last generated string to
|
|
||||||
# avoid duplicates (assuming the runs are sorted by time)
|
|
||||||
get_run_name.counter = 0
|
|
||||||
get_run_name.previous = ""
|
|
||||||
|
|
||||||
|
|
||||||
def reset_run_strings():
|
|
||||||
get_run_name.counter = 0
|
|
||||||
get_run_name.previous = ""
|
|
||||||
|
|
||||||
|
|
||||||
# Update all the x-ranges from a dict of plots
|
|
||||||
def reset_x_range(plot, x_range):
|
|
||||||
plot.x_range.factors = x_range
|
|
||||||
|
|
||||||
if len(x_range) < max_ticks:
|
|
||||||
plot.xaxis.major_tick_line_color = "#000000"
|
|
||||||
plot.xaxis.minor_tick_line_color = "#000000"
|
|
||||||
|
|
||||||
plot.xaxis.major_label_text_font_size = "8pt"
|
|
||||||
|
|
||||||
else:
|
|
||||||
plot.xaxis.major_tick_line_color = None
|
|
||||||
plot.xaxis.minor_tick_line_color = None
|
|
||||||
|
|
||||||
plot.xaxis.major_label_text_font_size = "0pt"
|
|
||||||
|
|
||||||
|
|
||||||
# Return an array of booleans that indicate which elements are outliers
|
|
||||||
# (True means element is not an outlier and must be kept)
|
|
||||||
def detect_outliers(array, max_zscore=max_zscore):
|
|
||||||
if len(array) <= 2:
|
|
||||||
return [True] * len(array)
|
|
||||||
|
|
||||||
median = np.median(array)
|
|
||||||
std = np.std(array)
|
|
||||||
if std == 0:
|
|
||||||
return array
|
|
||||||
distance = abs(array - median)
|
|
||||||
# Array of booleans with elements to be filtered
|
|
||||||
outliers_array = distance < max_zscore * std
|
|
||||||
|
|
||||||
return outliers_array
|
|
||||||
|
|
||||||
|
|
||||||
def remove_outliers(array, outliers):
|
|
||||||
return list(compress(array, outliers))
|
|
||||||
|
|
||||||
|
|
||||||
def remove_boxplot_outliers(dict, outliers, prefix):
|
|
||||||
outliers = detect_outliers(dict["%s_max" % prefix])
|
|
||||||
|
|
||||||
dict["%s_x" % prefix] = remove_outliers(dict["%s_x" % prefix], outliers)
|
|
||||||
|
|
||||||
dict["%s_min" % prefix] = remove_outliers(
|
|
||||||
dict["%s_min" % prefix], outliers)
|
|
||||||
dict["%s_quantile25" % prefix] = remove_outliers(
|
|
||||||
dict["%s_quantile25" % prefix], outliers)
|
|
||||||
dict["%s_quantile50" % prefix] = remove_outliers(
|
|
||||||
dict["%s_quantile50" % prefix], outliers)
|
|
||||||
dict["%s_quantile75" % prefix] = remove_outliers(
|
|
||||||
dict["%s_quantile75" % prefix], outliers)
|
|
||||||
dict["%s_max" % prefix] = remove_outliers(
|
|
||||||
dict["%s_max" % prefix], outliers)
|
|
||||||
dict["%s_mu" % prefix] = remove_outliers(dict["%s_mu" % prefix], outliers)
|
|
||||||
|
|
||||||
dict["nsamples"] = remove_outliers(dict["nsamples"], outliers)
|
|
||||||
@ -1,570 +0,0 @@
|
|||||||
# Manage the view comparing the variables of a run
|
|
||||||
|
|
||||||
from math import pi
|
|
||||||
from functools import partial
|
|
||||||
|
|
||||||
import pandas as pd
|
|
||||||
import numpy as np
|
|
||||||
|
|
||||||
from bokeh.plotting import figure, curdoc
|
|
||||||
from bokeh.embed import components
|
|
||||||
from bokeh.models import Select, ColumnDataSource, Panel, Tabs, HoverTool,\
|
|
||||||
RadioButtonGroup, CheckboxGroup, CustomJS
|
|
||||||
|
|
||||||
import helper
|
|
||||||
import plot
|
|
||||||
|
|
||||||
|
|
||||||
##########################################################################
|
|
||||||
|
|
||||||
|
|
||||||
class InspectRuns:
|
|
||||||
|
|
||||||
# Helper functions related to InspectRun
|
|
||||||
|
|
||||||
# Returns a dictionary mapping user-readable strings to all run timestamps
|
|
||||||
def gen_runs_selection(self):
|
|
||||||
|
|
||||||
runs_dict = {}
|
|
||||||
|
|
||||||
# Iterate over timestamp rows (runs) and fill dict
|
|
||||||
for row in self.metadata.iloc:
|
|
||||||
# The syntax used by pandas makes this part a bit tricky :
|
|
||||||
# row.name is the index of metadata (so it refers to the
|
|
||||||
# timestamp), whereas row["name"] is the column called "name"
|
|
||||||
# (which is the display string used for the run)
|
|
||||||
|
|
||||||
# runs_dict[run's name] = run's timestamp
|
|
||||||
runs_dict[row["name"]] = row.name
|
|
||||||
|
|
||||||
return runs_dict
|
|
||||||
|
|
||||||
def gen_boxplot_tooltips(self, prefix):
|
|
||||||
return [
|
|
||||||
("Name", "@%s_x" % prefix),
|
|
||||||
("Min", "@" + prefix + "_min{%0.18e}"),
|
|
||||||
("Max", "@" + prefix + "_max{%0.18e}"),
|
|
||||||
("1st quartile", "@" + prefix + "_quantile25{%0.18e}"),
|
|
||||||
("Median", "@" + prefix + "_quantile50{%0.18e}"),
|
|
||||||
("3rd quartile", "@" + prefix + "_quantile75{%0.18e}"),
|
|
||||||
("μ", "@" + prefix + "_mu{%0.18e}"),
|
|
||||||
("Number of samples (tests)", "@nsamples")
|
|
||||||
]
|
|
||||||
|
|
||||||
def gen_boxplot_tooltips_formatters(self, prefix):
|
|
||||||
return {
|
|
||||||
"@%s_min" % prefix: "printf",
|
|
||||||
"@%s_max" % prefix: "printf",
|
|
||||||
"@%s_quantile25" % prefix: "printf",
|
|
||||||
"@%s_quantile50" % prefix: "printf",
|
|
||||||
"@%s_quantile75" % prefix: "printf",
|
|
||||||
"@%s_mu" % prefix: "printf"
|
|
||||||
}
|
|
||||||
|
|
||||||
# Data processing helper
|
|
||||||
# (computes new distributions for sigma, s2, s10)
|
|
||||||
|
|
||||||
def data_processing(self, dataframe):
|
|
||||||
|
|
||||||
# Compute aggragated mu
|
|
||||||
dataframe["mu"] = np.vectorize(
|
|
||||||
np.average)(
|
|
||||||
dataframe["mu"],
|
|
||||||
weights=dataframe["nsamples"])
|
|
||||||
|
|
||||||
# nsamples is the number of aggregated elements (as well as the number
|
|
||||||
# of samples for our new sigma and s distributions)
|
|
||||||
dataframe["nsamples"] = dataframe["nsamples"].apply(lambda x: len(x))
|
|
||||||
|
|
||||||
dataframe["mu_x"] = dataframe.index
|
|
||||||
# Make sure that strings don't excede a certain length
|
|
||||||
dataframe["mu_x"] = dataframe["mu_x"].apply(
|
|
||||||
lambda x: x[:17] + "[...]" + x[-17:] if len(x) > 39 else x
|
|
||||||
)
|
|
||||||
|
|
||||||
# Get quantiles and mu for sigma, s10, s2
|
|
||||||
for prefix in ["sigma", "s10", "s2"]:
|
|
||||||
|
|
||||||
dataframe["%s_x" % prefix] = dataframe["mu_x"]
|
|
||||||
|
|
||||||
dataframe[prefix] = dataframe[prefix].apply(np.sort)
|
|
||||||
|
|
||||||
dataframe["%s_min" % prefix] = dataframe[prefix].apply(np.min)
|
|
||||||
dataframe["%s_quantile25" % prefix] = dataframe[prefix].apply(
|
|
||||||
np.quantile, args=(0.25,))
|
|
||||||
dataframe["%s_quantile50" % prefix] = dataframe[prefix].apply(
|
|
||||||
np.quantile, args=(0.50,))
|
|
||||||
dataframe["%s_quantile75" % prefix] = dataframe[prefix].apply(
|
|
||||||
np.quantile, args=(0.75,))
|
|
||||||
dataframe["%s_max" % prefix] = dataframe[prefix].apply(np.max)
|
|
||||||
dataframe["%s_mu" % prefix] = dataframe[prefix].apply(np.average)
|
|
||||||
del dataframe[prefix]
|
|
||||||
|
|
||||||
return dataframe
|
|
||||||
|
|
||||||
# Plots update function
|
|
||||||
|
|
||||||
def update_plots(self):
|
|
||||||
|
|
||||||
groupby_display = self.widgets["groupby_radio"].labels[
|
|
||||||
self.widgets["groupby_radio"].active
|
|
||||||
]
|
|
||||||
groupby = self.factors_dict[groupby_display]
|
|
||||||
|
|
||||||
filterby_display = self.widgets["filterby_radio"].labels[
|
|
||||||
self.widgets["filterby_radio"].active
|
|
||||||
]
|
|
||||||
filterby = self.factors_dict[filterby_display]
|
|
||||||
|
|
||||||
# Groupby and aggregate lines belonging to the same group in lists
|
|
||||||
|
|
||||||
groups = self.run_data[
|
|
||||||
self.run_data.index.isin(
|
|
||||||
[self.widgets["select_filter"].value],
|
|
||||||
level=filterby
|
|
||||||
)
|
|
||||||
].groupby(groupby)
|
|
||||||
|
|
||||||
groups = groups.agg({
|
|
||||||
"sigma": lambda x: x.tolist(),
|
|
||||||
"s10": lambda x: x.tolist(),
|
|
||||||
"s2": lambda x: x.tolist(),
|
|
||||||
|
|
||||||
"mu": lambda x: x.tolist(),
|
|
||||||
|
|
||||||
# Used for mu weighted average first, then will be replaced
|
|
||||||
"nsamples": lambda x: x.tolist()
|
|
||||||
})
|
|
||||||
|
|
||||||
# Compute the new distributions, ...
|
|
||||||
groups = self.data_processing(groups).to_dict("list")
|
|
||||||
|
|
||||||
# Update source
|
|
||||||
|
|
||||||
# Assign each ColumnDataSource, starting with the boxplots
|
|
||||||
for prefix in ["sigma", "s10", "s2"]:
|
|
||||||
|
|
||||||
dict = {
|
|
||||||
"%s_x" % prefix: groups["%s_x" % prefix],
|
|
||||||
"%s_min" % prefix: groups["%s_min" % prefix],
|
|
||||||
"%s_quantile25" % prefix: groups["%s_quantile25" % prefix],
|
|
||||||
"%s_quantile50" % prefix: groups["%s_quantile50" % prefix],
|
|
||||||
"%s_quantile75" % prefix: groups["%s_quantile75" % prefix],
|
|
||||||
"%s_max" % prefix: groups["%s_max" % prefix],
|
|
||||||
"%s_mu" % prefix: groups["%s_mu" % prefix],
|
|
||||||
|
|
||||||
"nsamples": groups["nsamples"]
|
|
||||||
}
|
|
||||||
|
|
||||||
# Filter outliers if the box is checked
|
|
||||||
if len(self.widgets["outliers_filtering_inspect"].active) > 0:
|
|
||||||
|
|
||||||
# Boxplots will be filtered by max then min
|
|
||||||
top_outliers = helper.detect_outliers(dict["%s_max" % prefix])
|
|
||||||
helper.remove_boxplot_outliers(dict, top_outliers, prefix)
|
|
||||||
|
|
||||||
bottom_outliers = helper.detect_outliers(
|
|
||||||
dict["%s_min" % prefix])
|
|
||||||
helper.remove_boxplot_outliers(dict, bottom_outliers, prefix)
|
|
||||||
|
|
||||||
self.sources["%s_source" % prefix].data = dict
|
|
||||||
|
|
||||||
# Finish with the mu plot
|
|
||||||
dict = {
|
|
||||||
"mu_x": groups["mu_x"],
|
|
||||||
"mu": groups["mu"],
|
|
||||||
|
|
||||||
"nsamples": groups["nsamples"]
|
|
||||||
}
|
|
||||||
|
|
||||||
self.sources["mu_source"].data = dict
|
|
||||||
|
|
||||||
# Filter outliers if the box is checked
|
|
||||||
if len(self.widgets["outliers_filtering_inspect"].active) > 0:
|
|
||||||
mu_outliers = helper.detect_outliers(groups["mu"])
|
|
||||||
groups["mu"] = helper.remove_outliers(groups["mu"], mu_outliers)
|
|
||||||
groups["mu_x"] = helper.remove_outliers(
|
|
||||||
groups["mu_x"], mu_outliers)
|
|
||||||
|
|
||||||
# Update plots axis/titles
|
|
||||||
|
|
||||||
# Get display string of the last (unselected) factor
|
|
||||||
factors_dict = self.factors_dict.copy()
|
|
||||||
del factors_dict[groupby_display]
|
|
||||||
del factors_dict[filterby_display]
|
|
||||||
for_all = list(factors_dict.keys())[0]
|
|
||||||
|
|
||||||
# Update all display strings for plot title (remove caps, plural)
|
|
||||||
groupby_display = groupby_display.lower()
|
|
||||||
filterby_display = filterby_display.lower()[:-1]
|
|
||||||
for_all = for_all.lower()
|
|
||||||
|
|
||||||
self.plots["mu_inspect"].title.text = \
|
|
||||||
"Empirical average μ of %s (groupped by %s, for all %s)" \
|
|
||||||
% (filterby_display, groupby_display, for_all)
|
|
||||||
|
|
||||||
self.plots["sigma_inspect"].title.text = \
|
|
||||||
"Standard deviation σ of %s (groupped by %s, for all %s)" \
|
|
||||||
% (filterby_display, groupby_display, for_all)
|
|
||||||
|
|
||||||
self.plots["s10_inspect"].title.text = \
|
|
||||||
"Significant digits s of %s (groupped by %s, for all %s)" \
|
|
||||||
% (filterby_display, groupby_display, for_all)
|
|
||||||
|
|
||||||
self.plots["s2_inspect"].title.text = \
|
|
||||||
"Significant digits s of %s (groupped by %s, for all %s)" \
|
|
||||||
% (filterby_display, groupby_display, for_all)
|
|
||||||
|
|
||||||
helper.reset_x_range(self.plots["mu_inspect"], groups["mu_x"])
|
|
||||||
helper.reset_x_range(self.plots["sigma_inspect"], groups["sigma_x"])
|
|
||||||
helper.reset_x_range(self.plots["s10_inspect"], groups["s10_x"])
|
|
||||||
helper.reset_x_range(self.plots["s2_inspect"], groups["s2_x"])
|
|
||||||
|
|
||||||
# Widets' callback functions
|
|
||||||
|
|
||||||
# Run selector callback
|
|
||||||
|
|
||||||
def update_run(self, attrname, old, new):
|
|
||||||
|
|
||||||
filterby = self.widgets["filterby_radio"].labels[
|
|
||||||
self.widgets["filterby_radio"].active
|
|
||||||
]
|
|
||||||
filterby = self.factors_dict[filterby]
|
|
||||||
|
|
||||||
# Update run selection (by using dict mapping)
|
|
||||||
self.current_run = self.runs_dict[new]
|
|
||||||
|
|
||||||
# Update run data
|
|
||||||
self.run_data = self.data[self.data["timestamp"] == self.current_run]
|
|
||||||
|
|
||||||
# Save old selected option
|
|
||||||
old_value = self.widgets["select_filter"].value
|
|
||||||
|
|
||||||
# Update filter options
|
|
||||||
options = self.run_data.index\
|
|
||||||
.get_level_values(filterby).drop_duplicates().tolist()
|
|
||||||
self.widgets["select_filter"].options = options
|
|
||||||
|
|
||||||
if old_value not in self.widgets["select_filter"].options:
|
|
||||||
self.widgets["select_filter"].value = options[0]
|
|
||||||
# The update_var callback will be triggered by the assignment
|
|
||||||
|
|
||||||
else:
|
|
||||||
# Trigger the callback manually (since the plots need to be updated
|
|
||||||
# anyway)
|
|
||||||
self.update_filter("", "", old_value)
|
|
||||||
|
|
||||||
# "Group by" radio
|
|
||||||
|
|
||||||
def update_groupby(self, attrname, old, new):
|
|
||||||
|
|
||||||
# Update "Filter by" radio list
|
|
||||||
filterby_list = list(self.factors_dict.keys())
|
|
||||||
del filterby_list[self.widgets["groupby_radio"].active]
|
|
||||||
self.widgets["filterby_radio"].labels = filterby_list
|
|
||||||
|
|
||||||
filterby = self.widgets["filterby_radio"].labels[
|
|
||||||
self.widgets["filterby_radio"].active
|
|
||||||
]
|
|
||||||
filterby = self.factors_dict[filterby]
|
|
||||||
|
|
||||||
# Save old selected option
|
|
||||||
old_value = self.widgets["select_filter"].value
|
|
||||||
|
|
||||||
# Update filter options
|
|
||||||
options = self.run_data.index\
|
|
||||||
.get_level_values(filterby).drop_duplicates().tolist()
|
|
||||||
self.widgets["select_filter"].options = options
|
|
||||||
|
|
||||||
if old_value not in self.widgets["select_filter"].options:
|
|
||||||
self.widgets["select_filter"].value = options[0]
|
|
||||||
# The update_var callback will be triggered by the assignment
|
|
||||||
|
|
||||||
else:
|
|
||||||
# Trigger the callback manually (since the plots need to be updated
|
|
||||||
# anyway)
|
|
||||||
self.update_filter("", "", old_value)
|
|
||||||
|
|
||||||
# "Filter by" radio
|
|
||||||
|
|
||||||
def update_filterby(self, attrname, old, new):
|
|
||||||
|
|
||||||
filterby = self.widgets["filterby_radio"].labels[
|
|
||||||
self.widgets["filterby_radio"].active
|
|
||||||
]
|
|
||||||
filterby = self.factors_dict[filterby]
|
|
||||||
|
|
||||||
# Save old selected option
|
|
||||||
old_value = self.widgets["select_filter"].value
|
|
||||||
|
|
||||||
# Update filter selector options
|
|
||||||
options = self.run_data.index\
|
|
||||||
.get_level_values(filterby).drop_duplicates().tolist()
|
|
||||||
self.widgets["select_filter"].options = options
|
|
||||||
|
|
||||||
if old_value not in self.widgets["select_filter"].options:
|
|
||||||
self.widgets["select_filter"].value = options[0]
|
|
||||||
# The update_var callback will be triggered by the assignment
|
|
||||||
|
|
||||||
else:
|
|
||||||
# Trigger the callback manually (since the plots need to be updated
|
|
||||||
# anyway)
|
|
||||||
self.update_filter("", "", old_value)
|
|
||||||
|
|
||||||
# Filter selector callback
|
|
||||||
|
|
||||||
def update_filter(self, attrname, old, new):
|
|
||||||
self.update_plots()
|
|
||||||
|
|
||||||
# Filter outliers checkbox callback
|
|
||||||
|
|
||||||
def update_outliers_filtering(self, attrname, old, new):
|
|
||||||
# The status (checked/unchecked) of the checkbox is also verified inside
|
|
||||||
# self.update_plots(), so calling this function is enough
|
|
||||||
self.update_plots()
|
|
||||||
|
|
||||||
# Bokeh setup functions
|
|
||||||
# (for both variable and backend selection at once)
|
|
||||||
|
|
||||||
def setup_plots(self):
|
|
||||||
|
|
||||||
tools = "pan, wheel_zoom, xwheel_zoom, ywheel_zoom, reset, save"
|
|
||||||
|
|
||||||
# Tooltips and formatters
|
|
||||||
|
|
||||||
dotplot_tooltips = [
|
|
||||||
("Name", "@mu_x"),
|
|
||||||
("μ", "@mu{%0.18e}"),
|
|
||||||
("Number of samples (tests)", "@nsamples")
|
|
||||||
]
|
|
||||||
dotplot_formatters = {
|
|
||||||
"@mu": "printf"
|
|
||||||
}
|
|
||||||
|
|
||||||
sigma_boxplot_tooltips = self.gen_boxplot_tooltips("sigma")
|
|
||||||
sigma_boxplot_tooltips_formatters = self.gen_boxplot_tooltips_formatters(
|
|
||||||
"sigma")
|
|
||||||
|
|
||||||
s10_boxplot_tooltips = self.gen_boxplot_tooltips("s10")
|
|
||||||
s10_boxplot_tooltips_formatters = self.gen_boxplot_tooltips_formatters(
|
|
||||||
"s10")
|
|
||||||
|
|
||||||
s2_boxplot_tooltips = self.gen_boxplot_tooltips("s2")
|
|
||||||
s2_boxplot_tooltips_formatters = self.gen_boxplot_tooltips_formatters(
|
|
||||||
"s2")
|
|
||||||
|
|
||||||
# Plots
|
|
||||||
|
|
||||||
# Mu plot
|
|
||||||
self.plots["mu_inspect"] = figure(
|
|
||||||
name="mu_inspect",
|
|
||||||
title="",
|
|
||||||
plot_width=900, plot_height=400, x_range=[""],
|
|
||||||
tools=tools, sizing_mode="scale_width"
|
|
||||||
)
|
|
||||||
plot.fill_dotplot(
|
|
||||||
self.plots["mu_inspect"], self.sources["mu_source"], "mu",
|
|
||||||
tooltips=dotplot_tooltips,
|
|
||||||
tooltips_formatters=dotplot_formatters
|
|
||||||
)
|
|
||||||
self.doc.add_root(self.plots["mu_inspect"])
|
|
||||||
|
|
||||||
# Sigma plot
|
|
||||||
self.plots["sigma_inspect"] = figure(
|
|
||||||
name="sigma_inspect",
|
|
||||||
title="",
|
|
||||||
plot_width=900, plot_height=400, x_range=[""],
|
|
||||||
tools=tools, sizing_mode="scale_width"
|
|
||||||
)
|
|
||||||
plot.fill_boxplot(
|
|
||||||
self.plots["sigma_inspect"],
|
|
||||||
self.sources["sigma_source"],
|
|
||||||
prefix="sigma",
|
|
||||||
tooltips=sigma_boxplot_tooltips,
|
|
||||||
tooltips_formatters=sigma_boxplot_tooltips_formatters)
|
|
||||||
self.doc.add_root(self.plots["sigma_inspect"])
|
|
||||||
|
|
||||||
# s plots
|
|
||||||
self.plots["s10_inspect"] = figure(
|
|
||||||
name="s10_inspect",
|
|
||||||
title="",
|
|
||||||
plot_width=900, plot_height=400, x_range=[""],
|
|
||||||
tools=tools, sizing_mode='scale_width'
|
|
||||||
)
|
|
||||||
plot.fill_boxplot(
|
|
||||||
self.plots["s10_inspect"],
|
|
||||||
self.sources["s10_source"],
|
|
||||||
prefix="s10",
|
|
||||||
tooltips=s10_boxplot_tooltips,
|
|
||||||
tooltips_formatters=s10_boxplot_tooltips_formatters)
|
|
||||||
s10_tab_inspect = Panel(
|
|
||||||
child=self.plots["s10_inspect"],
|
|
||||||
title="Base 10")
|
|
||||||
|
|
||||||
self.plots["s2_inspect"] = figure(
|
|
||||||
name="s2_inspect",
|
|
||||||
title="",
|
|
||||||
plot_width=900, plot_height=400, x_range=[""],
|
|
||||||
tools=tools, sizing_mode='scale_width'
|
|
||||||
)
|
|
||||||
plot.fill_boxplot(
|
|
||||||
self.plots["s2_inspect"], self.sources["s2_source"], prefix="s2",
|
|
||||||
tooltips=s2_boxplot_tooltips,
|
|
||||||
tooltips_formatters=s2_boxplot_tooltips_formatters
|
|
||||||
)
|
|
||||||
s2_tab_inspect = Panel(child=self.plots["s2_inspect"], title="Base 2")
|
|
||||||
|
|
||||||
s_tabs_inspect = Tabs(
|
|
||||||
name="s_tabs_inspect",
|
|
||||||
tabs=[s10_tab_inspect, s2_tab_inspect], tabs_location="below"
|
|
||||||
)
|
|
||||||
self.doc.add_root(s_tabs_inspect)
|
|
||||||
|
|
||||||
def setup_widgets(self):
|
|
||||||
|
|
||||||
# Generation of selectable items
|
|
||||||
|
|
||||||
# Dict contains all inspectable runs (maps display strings to timestamps)
|
|
||||||
# The dict structure allows to get the timestamp from the display string
|
|
||||||
# in O(1)
|
|
||||||
self.runs_dict = self.gen_runs_selection()
|
|
||||||
|
|
||||||
# Dict maps display strings to column names for the different factors
|
|
||||||
# (var, backend, test)
|
|
||||||
self.factors_dict = {
|
|
||||||
"Variables": "variable",
|
|
||||||
"Backends": "vfc_backend",
|
|
||||||
"Tests": "test"
|
|
||||||
}
|
|
||||||
|
|
||||||
# Run selection
|
|
||||||
|
|
||||||
# Contains all options strings
|
|
||||||
runs_display = list(self.runs_dict.keys())
|
|
||||||
# Will be used when updating plots (contains actual number)
|
|
||||||
self.current_run = self.runs_dict[runs_display[-1]]
|
|
||||||
# Contains the selected option string, used to update current_n_runs
|
|
||||||
current_run_display = runs_display[-1]
|
|
||||||
# This contains only entries matching the run
|
|
||||||
self.run_data = self.data[self.data["timestamp"] == self.current_run]
|
|
||||||
|
|
||||||
change_run_callback_js = "updateRunMetadata(cb_obj.value);"
|
|
||||||
|
|
||||||
self.widgets["select_run"] = Select(
|
|
||||||
name="select_run", title="Run :",
|
|
||||||
value=current_run_display, options=runs_display
|
|
||||||
)
|
|
||||||
self.doc.add_root(self.widgets["select_run"])
|
|
||||||
self.widgets["select_run"].on_change("value", self.update_run)
|
|
||||||
self.widgets["select_run"].js_on_change("value", CustomJS(
|
|
||||||
code=change_run_callback_js,
|
|
||||||
args=(dict(
|
|
||||||
metadata=helper.metadata_to_dict(
|
|
||||||
helper.get_metadata(self.metadata, self.current_run)
|
|
||||||
)
|
|
||||||
))
|
|
||||||
))
|
|
||||||
|
|
||||||
# Factors selection
|
|
||||||
|
|
||||||
# "Group by" radio
|
|
||||||
self.widgets["groupby_radio"] = RadioButtonGroup(
|
|
||||||
name="groupby_radio",
|
|
||||||
labels=list(self.factors_dict.keys()), active=0
|
|
||||||
)
|
|
||||||
self.doc.add_root(self.widgets["groupby_radio"])
|
|
||||||
# The functions are defined inside the template to avoid writing too
|
|
||||||
# much JS server side
|
|
||||||
self.widgets["groupby_radio"].on_change(
|
|
||||||
"active",
|
|
||||||
self.update_groupby
|
|
||||||
)
|
|
||||||
|
|
||||||
# "Filter by" radio
|
|
||||||
# Get all possible factors, and remove the one selected in "Group by"
|
|
||||||
filterby_list = list(self.factors_dict.keys())
|
|
||||||
del filterby_list[self.widgets["groupby_radio"].active]
|
|
||||||
|
|
||||||
self.widgets["filterby_radio"] = RadioButtonGroup(
|
|
||||||
name="filterby_radio",
|
|
||||||
labels=filterby_list, active=0
|
|
||||||
)
|
|
||||||
self.doc.add_root(self.widgets["filterby_radio"])
|
|
||||||
# The functions are defined inside the template to avoid writing too
|
|
||||||
# much JS server side
|
|
||||||
self.widgets["filterby_radio"].on_change(
|
|
||||||
"active",
|
|
||||||
self.update_filterby
|
|
||||||
)
|
|
||||||
|
|
||||||
# Filter selector
|
|
||||||
|
|
||||||
filterby = self.widgets["filterby_radio"].labels[
|
|
||||||
self.widgets["filterby_radio"].active
|
|
||||||
]
|
|
||||||
filterby = self.factors_dict[filterby]
|
|
||||||
|
|
||||||
options = self.run_data.index\
|
|
||||||
.get_level_values(filterby).drop_duplicates().tolist()
|
|
||||||
|
|
||||||
self.widgets["select_filter"] = Select(
|
|
||||||
# We need a different name to avoid collision in the template with
|
|
||||||
# the runs comparison's widget
|
|
||||||
name="select_filter", title="Select a filter :",
|
|
||||||
value=options[0], options=options
|
|
||||||
)
|
|
||||||
self.doc.add_root(self.widgets["select_filter"])
|
|
||||||
self.widgets["select_filter"]\
|
|
||||||
.on_change("value", self.update_filter)
|
|
||||||
|
|
||||||
# Toggle for outliers filtering
|
|
||||||
|
|
||||||
self.widgets["outliers_filtering_inspect"] = CheckboxGroup(
|
|
||||||
name="outliers_filtering_inspect",
|
|
||||||
labels=["Filter outliers"], active=[]
|
|
||||||
)
|
|
||||||
self.doc.add_root(self.widgets["outliers_filtering_inspect"])
|
|
||||||
self.widgets["outliers_filtering_inspect"]\
|
|
||||||
.on_change("active", self.update_outliers_filtering)
|
|
||||||
|
|
||||||
# Communication methods
|
|
||||||
# (to send/receive messages to/from master)
|
|
||||||
|
|
||||||
# When received, switch to the run_name in parameter
|
|
||||||
|
|
||||||
def switch_view(self, run_name):
|
|
||||||
self.widgets["select_run"].value = run_name
|
|
||||||
|
|
||||||
# Constructor
|
|
||||||
|
|
||||||
def __init__(self, master, doc, data, metadata):
|
|
||||||
|
|
||||||
self.master = master
|
|
||||||
|
|
||||||
self.doc = doc
|
|
||||||
self.data = data
|
|
||||||
self.metadata = metadata
|
|
||||||
|
|
||||||
self.sources = {
|
|
||||||
"mu_source": ColumnDataSource(data={}),
|
|
||||||
"sigma_source": ColumnDataSource(data={}),
|
|
||||||
"s10_source": ColumnDataSource(data={}),
|
|
||||||
"s2_source": ColumnDataSource(data={})
|
|
||||||
}
|
|
||||||
|
|
||||||
self.plots = {}
|
|
||||||
self.widgets = {}
|
|
||||||
|
|
||||||
# Setup Bokeh objects
|
|
||||||
self.setup_plots()
|
|
||||||
self.setup_widgets()
|
|
||||||
|
|
||||||
# Pass the initial metadata to the template (will be updated in CustomJS
|
|
||||||
# callbacks). This is required because metadata is not displayed in a
|
|
||||||
# Bokeh widget, so we can't update this with a server callback.
|
|
||||||
initial_run = helper.get_metadata(self.metadata, self.current_run)
|
|
||||||
self.doc.template_variables["initial_timestamp"] = self.current_run
|
|
||||||
|
|
||||||
# At this point, everything should have been initialized, so we can
|
|
||||||
# show the plots for the first time
|
|
||||||
self.update_plots()
|
|
||||||
@ -1,208 +0,0 @@
|
|||||||
# Look for and read all the run files in the current directory (ending with
|
|
||||||
# .vfcrunh5), and lanch a Bokeh server for the visualization of this data.
|
|
||||||
|
|
||||||
import os
|
|
||||||
import sys
|
|
||||||
import time
|
|
||||||
|
|
||||||
import pandas as pd
|
|
||||||
|
|
||||||
from bokeh.plotting import curdoc
|
|
||||||
|
|
||||||
# Local imports from vfc_ci_server
|
|
||||||
import compare_runs
|
|
||||||
import inspect_runs
|
|
||||||
import helper
|
|
||||||
|
|
||||||
##########################################################################
|
|
||||||
|
|
||||||
# Read vfcrun files, and aggregate them in one dataset
|
|
||||||
|
|
||||||
run_files = [f for f in os.listdir(".") if f.endswith(".vfcrun.h5")]
|
|
||||||
|
|
||||||
if len(run_files) == 0:
|
|
||||||
print(
|
|
||||||
"Warning [vfc_ci]: Could not find any vfcrun files in the directory. "
|
|
||||||
"This will result in server errors and prevent you from viewing the report.")
|
|
||||||
|
|
||||||
# These are arrays of Pandas dataframes for now
|
|
||||||
metadata = []
|
|
||||||
data = []
|
|
||||||
|
|
||||||
for f in run_files:
|
|
||||||
metadata.append(pd.read_hdf(f, "metadata"))
|
|
||||||
data.append(pd.read_hdf(f, "data"))
|
|
||||||
|
|
||||||
metadata = pd.concat(metadata).sort_index()
|
|
||||||
data = pd.concat(data).sort_index()
|
|
||||||
|
|
||||||
|
|
||||||
# Generate the display strings for runs (runs ticks)
|
|
||||||
# By doing this in master, we ensure the homogeneity of display strings
|
|
||||||
# across all plots
|
|
||||||
metadata["name"] = metadata.index.to_series().map(
|
|
||||||
lambda x: helper.get_run_name(
|
|
||||||
x,
|
|
||||||
helper.get_metadata(metadata, x)["hash"]
|
|
||||||
)
|
|
||||||
)
|
|
||||||
helper.reset_run_strings()
|
|
||||||
|
|
||||||
metadata["date"] = metadata.index.to_series().map(
|
|
||||||
lambda x: time.ctime(x)
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
##########################################################################
|
|
||||||
|
|
||||||
|
|
||||||
curdoc().title = "Verificarlo Report"
|
|
||||||
|
|
||||||
# Read server arguments
|
|
||||||
# (this is quite easy because Bokeh server is called through a wrapper, so
|
|
||||||
# we know exactly what the arguments might be)
|
|
||||||
|
|
||||||
git_repo_linked = False
|
|
||||||
commit_link = ""
|
|
||||||
|
|
||||||
has_logo = False
|
|
||||||
logo_url = ""
|
|
||||||
|
|
||||||
for i in range(1, len(sys.argv)):
|
|
||||||
|
|
||||||
# Look for the Git repository remote address
|
|
||||||
# (if a Git repo is specified, the webpage will contain hyperlinks to the
|
|
||||||
# repository and the different commits)
|
|
||||||
if sys.argv[i] == "git":
|
|
||||||
from urllib.parse import urlparse
|
|
||||||
|
|
||||||
method = sys.argv[i + 1]
|
|
||||||
address = sys.argv[i + 2]
|
|
||||||
url = ""
|
|
||||||
|
|
||||||
# Here, address is either the remote URL or the path to the local Git
|
|
||||||
# repo (depending on the method)
|
|
||||||
|
|
||||||
if method == "url":
|
|
||||||
# We should directly have a Git URL
|
|
||||||
url = address
|
|
||||||
|
|
||||||
elif method == "directory":
|
|
||||||
# Get the remote URL from the local repo
|
|
||||||
from git import Repo
|
|
||||||
repo = Repo(address)
|
|
||||||
url = repo.remotes.origin.url
|
|
||||||
|
|
||||||
else:
|
|
||||||
raise ValueError(
|
|
||||||
"Error [vfc_ci]: The specified method to get the Git "
|
|
||||||
"repository is invalid. Are you calling Bokeh directly "
|
|
||||||
"instead of using the Verificarlo wrapper ?"
|
|
||||||
)
|
|
||||||
|
|
||||||
# At this point, "url" should be set correctly, we can get the repo's
|
|
||||||
# URL and name, after making sure we're on a Git URL
|
|
||||||
|
|
||||||
parsed_url = urlparse(url)
|
|
||||||
|
|
||||||
path = parsed_url.path.split("/")
|
|
||||||
if len(path) < 3:
|
|
||||||
raise ValueError(
|
|
||||||
"Error [vfc_ci]: The found URL doesn't seem to be pointing "
|
|
||||||
"to a Git repository (path is too short)"
|
|
||||||
)
|
|
||||||
|
|
||||||
repo_name = path[2]
|
|
||||||
|
|
||||||
curdoc().template_variables["repo_url"] = url
|
|
||||||
curdoc().template_variables["repo_name"] = repo_name
|
|
||||||
|
|
||||||
# We should have a "github.com" or a "*gitlab*" URL
|
|
||||||
|
|
||||||
if parsed_url.netloc == "github.com":
|
|
||||||
commit_link = "https://%s%s/commit/" \
|
|
||||||
% (parsed_url.netloc, parsed_url.path)
|
|
||||||
|
|
||||||
curdoc().template_variables["commit_link"] = commit_link
|
|
||||||
curdoc().template_variables["git_host"] = "GitHub"
|
|
||||||
|
|
||||||
# Used in Bokeh tooltips
|
|
||||||
commit_link = commit_link + "@hash"
|
|
||||||
|
|
||||||
# We assume we have a GitLab URL
|
|
||||||
else:
|
|
||||||
commit_link = "https://%s%s/-/commit/" \
|
|
||||||
% (parsed_url.netloc, parsed_url.path)
|
|
||||||
|
|
||||||
curdoc().template_variables["commit_link"] = commit_link
|
|
||||||
curdoc().template_variables["git_host"] = "GitLab"
|
|
||||||
|
|
||||||
# Used in Bokeh tooltips
|
|
||||||
commit_link = commit_link + "@hash"
|
|
||||||
|
|
||||||
git_repo_linked = True
|
|
||||||
|
|
||||||
# Look for a logo URL
|
|
||||||
# If a logo URL is specified, it will be included in the report's header
|
|
||||||
if sys.argv[i] == "logo":
|
|
||||||
curdoc().template_variables["logo_url"] = sys.argv[i + 1]
|
|
||||||
has_logo = True
|
|
||||||
|
|
||||||
|
|
||||||
# After the loop, we know if a repo has been linked, if we have a logo, ...
|
|
||||||
curdoc().template_variables["git_repo_linked"] = git_repo_linked
|
|
||||||
curdoc().template_variables["has_logo"] = has_logo
|
|
||||||
|
|
||||||
|
|
||||||
##########################################################################
|
|
||||||
|
|
||||||
# Setup report views
|
|
||||||
|
|
||||||
# Define a ViewsMaster class to allow two-ways communication between views.
|
|
||||||
# This approach by classes allows us to have separate scopes for each view and
|
|
||||||
# will be useful if we want to add new views at some point in the future
|
|
||||||
# (instead of having n views with n-1 references each).
|
|
||||||
|
|
||||||
class ViewsMaster:
|
|
||||||
|
|
||||||
# Communication functions
|
|
||||||
|
|
||||||
def go_to_inspect(self, run_name):
|
|
||||||
self.inspect.switch_view(run_name)
|
|
||||||
|
|
||||||
# Constructor
|
|
||||||
|
|
||||||
def __init__(self, data, metadata, git_repo_linked, commit_link):
|
|
||||||
|
|
||||||
self.data = data
|
|
||||||
self.metadata = metadata
|
|
||||||
self.git_repo_linked = git_repo_linked
|
|
||||||
self.commit_link = commit_link
|
|
||||||
|
|
||||||
# Pass metadata to the template as a JSON string
|
|
||||||
curdoc().template_variables["metadata"] = self.metadata.to_json(
|
|
||||||
orient="index")
|
|
||||||
|
|
||||||
# Runs comparison
|
|
||||||
self.compare = compare_runs.CompareRuns(
|
|
||||||
master=self,
|
|
||||||
doc=curdoc(),
|
|
||||||
data=data,
|
|
||||||
metadata=metadata,
|
|
||||||
)
|
|
||||||
|
|
||||||
# Runs inspection
|
|
||||||
self.inspect = inspect_runs.InspectRuns(
|
|
||||||
master=self,
|
|
||||||
doc=curdoc(),
|
|
||||||
data=data,
|
|
||||||
metadata=metadata,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
views_master = ViewsMaster(
|
|
||||||
data=data,
|
|
||||||
metadata=metadata,
|
|
||||||
git_repo_linked=git_repo_linked,
|
|
||||||
commit_link=commit_link
|
|
||||||
)
|
|
||||||
@ -1,142 +0,0 @@
|
|||||||
# General functions for filling plots with data in all report's views
|
|
||||||
|
|
||||||
from bokeh.plotting import figure
|
|
||||||
from bokeh.models import HoverTool, TapTool, CustomJS
|
|
||||||
|
|
||||||
from math import pi
|
|
||||||
|
|
||||||
|
|
||||||
def fill_dotplot(
|
|
||||||
plot, source, data_field,
|
|
||||||
tooltips=None, tooltips_formatters=None,
|
|
||||||
js_tap_callback=None, server_tap_callback=None,
|
|
||||||
lines=False,
|
|
||||||
lower_bound=False
|
|
||||||
):
|
|
||||||
|
|
||||||
# (Optional) Tooltip and tooltip formatters
|
|
||||||
if tooltips is not None:
|
|
||||||
hover = HoverTool(tooltips=tooltips, mode="vline", names=["circle"])
|
|
||||||
|
|
||||||
if tooltips_formatters is not None:
|
|
||||||
hover.formatters = tooltips_formatters
|
|
||||||
|
|
||||||
plot.add_tools(hover)
|
|
||||||
|
|
||||||
# (Optional) Add TapTool (for JS tap callback)
|
|
||||||
if js_tap_callback is not None:
|
|
||||||
tap = TapTool(callback=CustomJS(code=js_tap_callback))
|
|
||||||
plot.add_tools(tap)
|
|
||||||
|
|
||||||
# (Optional) Add segment to represent a lower bound
|
|
||||||
if lower_bound:
|
|
||||||
lower_segment = plot.segment(
|
|
||||||
x0="%s_x" % data_field, y0=data_field,
|
|
||||||
x1="%s_x" % data_field, y1="%s_lower_bound" % data_field,
|
|
||||||
source=source, line_color="black"
|
|
||||||
)
|
|
||||||
|
|
||||||
# Draw dots (actually Bokeh circles)
|
|
||||||
circle = plot.circle(
|
|
||||||
name="circle",
|
|
||||||
x="%s_x" % data_field, y=data_field, source=source, size=12
|
|
||||||
)
|
|
||||||
|
|
||||||
# (Optional) Draw lines between dots
|
|
||||||
if lines:
|
|
||||||
line = plot.line(x="%s_x" % data_field, y=data_field, source=source)
|
|
||||||
|
|
||||||
# (Optional) Add server tap callback
|
|
||||||
if server_tap_callback is not None:
|
|
||||||
circle.data_source.selected.on_change("indices", server_tap_callback)
|
|
||||||
|
|
||||||
# Plot appearance
|
|
||||||
plot.xgrid.grid_line_color = None
|
|
||||||
plot.ygrid.grid_line_color = None
|
|
||||||
|
|
||||||
plot.yaxis[0].formatter.power_limit_high = 0
|
|
||||||
plot.yaxis[0].formatter.power_limit_low = 0
|
|
||||||
plot.yaxis[0].formatter.precision = 3
|
|
||||||
|
|
||||||
plot.xaxis[0].major_label_orientation = pi / 8
|
|
||||||
|
|
||||||
|
|
||||||
def fill_boxplot(
|
|
||||||
plot, source,
|
|
||||||
prefix="",
|
|
||||||
tooltips=None, tooltips_formatters=None,
|
|
||||||
js_tap_callback=None, server_tap_callback=None
|
|
||||||
):
|
|
||||||
|
|
||||||
# (Optional) Tooltip and tooltip formatters
|
|
||||||
if tooltips is not None:
|
|
||||||
hover = HoverTool(tooltips=tooltips, mode="vline", names=["full_box"])
|
|
||||||
|
|
||||||
if tooltips_formatters is not None:
|
|
||||||
hover.formatters = tooltips_formatters
|
|
||||||
|
|
||||||
plot.add_tools(hover)
|
|
||||||
|
|
||||||
# (Optional) Add TapTool (for JS tap callback)
|
|
||||||
if js_tap_callback is not None:
|
|
||||||
tap = TapTool(callback=CustomJS(code=js_tap_callback))
|
|
||||||
plot.add_tools(tap)
|
|
||||||
|
|
||||||
# Draw boxes (the prefix argument modifies the fields of ColumnDataSource
|
|
||||||
# that are used)
|
|
||||||
|
|
||||||
if prefix != "":
|
|
||||||
prefix = "%s_" % prefix
|
|
||||||
|
|
||||||
# Stems
|
|
||||||
top_stem = plot.segment(
|
|
||||||
x0="%sx" % prefix, y0="%smax" % prefix,
|
|
||||||
x1="%sx" % prefix, y1="%squantile75" % prefix,
|
|
||||||
source=source, line_color="black"
|
|
||||||
)
|
|
||||||
bottom_stem = plot.segment(
|
|
||||||
x0="%sx" % prefix, y0="%smin" % prefix,
|
|
||||||
x1="%sx" % prefix, y1="%squantile25" % prefix,
|
|
||||||
source=source, line_color="black"
|
|
||||||
)
|
|
||||||
|
|
||||||
# Boxes
|
|
||||||
full_box = plot.vbar(
|
|
||||||
name="full_box",
|
|
||||||
x="%sx" % prefix, width=0.5,
|
|
||||||
top="%squantile75" % prefix, bottom="%squantile25" % prefix,
|
|
||||||
source=source, line_color="black"
|
|
||||||
)
|
|
||||||
bottom_box = plot.vbar(
|
|
||||||
x="%sx" % prefix, width=0.5,
|
|
||||||
top="%squantile50" % prefix, bottom="%squantile25" % prefix,
|
|
||||||
source=source, line_color="black"
|
|
||||||
)
|
|
||||||
|
|
||||||
# Mu dot
|
|
||||||
mu_dot = plot.dot(
|
|
||||||
x="%sx" % prefix, y="%smu" % prefix, size=30, source=source,
|
|
||||||
color="black"
|
|
||||||
)
|
|
||||||
|
|
||||||
# (Optional) Add server tap callback
|
|
||||||
if server_tap_callback is not None:
|
|
||||||
top_stem.data_source.selected.on_change("indices", server_tap_callback)
|
|
||||||
bottom_stem.data_source.selected.on_change(
|
|
||||||
"indices", server_tap_callback)
|
|
||||||
|
|
||||||
full_box.data_source.selected.on_change("indices", server_tap_callback)
|
|
||||||
bottom_box.data_source.selected.on_change(
|
|
||||||
"indices", server_tap_callback)
|
|
||||||
|
|
||||||
mu_dot.data_source.selected.on_change("indices", server_tap_callback)
|
|
||||||
|
|
||||||
# Plot appearance
|
|
||||||
plot.xgrid.grid_line_color = None
|
|
||||||
plot.ygrid.grid_line_color = None
|
|
||||||
|
|
||||||
plot.yaxis[0].formatter.power_limit_high = 0
|
|
||||||
plot.yaxis[0].formatter.power_limit_low = 0
|
|
||||||
plot.yaxis[0].formatter.precision = 3
|
|
||||||
|
|
||||||
plot.xaxis[0].major_label_orientation = pi / 8
|
|
||||||
@ -1,482 +0,0 @@
|
|||||||
<!DOCTYPE html>
|
|
||||||
<html>
|
|
||||||
|
|
||||||
<head>
|
|
||||||
<title>Verificarlo Report</title>
|
|
||||||
<meta charset="utf8">
|
|
||||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
||||||
|
|
||||||
<!-- This template uses Bulma for CSS : https://bulma.io/ -->
|
|
||||||
<!-- Doc : https://bulma.io/documentation/ -->
|
|
||||||
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bulma@0.9.2/css/bulma.min.css">
|
|
||||||
|
|
||||||
<style>
|
|
||||||
html, body{
|
|
||||||
background-color: #f5f5f5;
|
|
||||||
}
|
|
||||||
|
|
||||||
#navbar {
|
|
||||||
height: 67px;
|
|
||||||
}
|
|
||||||
|
|
||||||
#logo-link {
|
|
||||||
padding: 0;
|
|
||||||
padding-left: 16px;
|
|
||||||
}
|
|
||||||
|
|
||||||
#logo-img {
|
|
||||||
object-fit: cover;
|
|
||||||
margin-top: -18px;
|
|
||||||
margin-left: -8px;
|
|
||||||
max-height: 50px;
|
|
||||||
}
|
|
||||||
|
|
||||||
#compare-runs-container {
|
|
||||||
margin-top: 1em;
|
|
||||||
}
|
|
||||||
|
|
||||||
#inspect-runs-container {
|
|
||||||
margin-top: 1em;
|
|
||||||
display : none; /* This one is hidden by default */
|
|
||||||
}
|
|
||||||
|
|
||||||
.plot-card {
|
|
||||||
width: 900px;
|
|
||||||
}
|
|
||||||
|
|
||||||
#loader {
|
|
||||||
height: 100%;
|
|
||||||
}
|
|
||||||
|
|
||||||
#loading-logo {
|
|
||||||
position: absolute;
|
|
||||||
left: 50%;
|
|
||||||
top: 50%;
|
|
||||||
transform: translate(-50%, -50%);
|
|
||||||
max-width: 400px;
|
|
||||||
|
|
||||||
animation: pulse 1.5s linear infinite;
|
|
||||||
}
|
|
||||||
|
|
||||||
@keyframes pulse {
|
|
||||||
0% {
|
|
||||||
transform: scale(1) translate(-50%, -50%);
|
|
||||||
}
|
|
||||||
|
|
||||||
50% {
|
|
||||||
transform: scale(0.9) translate(-55%, -55%);
|
|
||||||
}
|
|
||||||
|
|
||||||
100% {
|
|
||||||
transform: scale(1) translate(-50%, -50%);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
</style>
|
|
||||||
|
|
||||||
{% extends base %}
|
|
||||||
</head>
|
|
||||||
|
|
||||||
|
|
||||||
<body>
|
|
||||||
{% block contents %}
|
|
||||||
|
|
||||||
|
|
||||||
<!-- REPORT -->
|
|
||||||
<div id="report" style="display: none;">
|
|
||||||
|
|
||||||
<!-- HEADER -->
|
|
||||||
<nav class="navbar has-shadow" id="navbar"
|
|
||||||
role="navigation" aria-label="navbar-content">
|
|
||||||
|
|
||||||
<!-- BRAND (left part) -->
|
|
||||||
<div class="navbar-brand">
|
|
||||||
{% if has_logo %}
|
|
||||||
<a
|
|
||||||
class="navbar-item" id="logo-link" href="."
|
|
||||||
style="margin-top: 12px;"
|
|
||||||
>
|
|
||||||
<img id="logo-img" width="85" height="45"
|
|
||||||
src="{{ logo_url }}">
|
|
||||||
</a>
|
|
||||||
{% endif %}
|
|
||||||
|
|
||||||
<a class="navbar-item" id="logo-link" href=".">
|
|
||||||
<img id="logo-img" width="85" height="45"
|
|
||||||
src="https://avatars1.githubusercontent.com/u/12033642">
|
|
||||||
</a>
|
|
||||||
|
|
||||||
<a role="button" class="navbar-burger" id="navbar-burger"
|
|
||||||
aria-label="menu" aria-expanded="false"
|
|
||||||
data-target="navbarBasicExample">
|
|
||||||
<span aria-hidden="true"></span>
|
|
||||||
<span aria-hidden="true"></span>
|
|
||||||
<span aria-hidden="true"></span>
|
|
||||||
</a>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<!-- MENU (content) -->
|
|
||||||
<div id="navbar-content" class="navbar-menu">
|
|
||||||
|
|
||||||
<div id="buttons-container" class="navbar-start">
|
|
||||||
<a class="navbar-item is-active" id="compare-runs-button">
|
|
||||||
Compare runs
|
|
||||||
</a>
|
|
||||||
|
|
||||||
<a class="navbar-item" id="inspect-runs-button">
|
|
||||||
Inspect runs
|
|
||||||
</a>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<div class="navbar-end">
|
|
||||||
{% if git_repo_linked %}
|
|
||||||
<div class="navbar-item">
|
|
||||||
<div class="buttons">
|
|
||||||
<a class="button is-light" href="{{repo_url}}" target="_blank">
|
|
||||||
{{repo_name}} on {{git_host}}
|
|
||||||
</a>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
{% endif %}
|
|
||||||
|
|
||||||
<div class="navbar-item">
|
|
||||||
<div class="buttons">
|
|
||||||
<a class="button is-light"
|
|
||||||
href="https://github.com/verificarlo" target="_blank">
|
|
||||||
Verificarlo on GitHub
|
|
||||||
</a>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
</div>
|
|
||||||
</nav>
|
|
||||||
|
|
||||||
|
|
||||||
<!-- CONTENT : COMPARE RUNS -->
|
|
||||||
<main class="container" id="compare-runs-container">
|
|
||||||
<div class="columns">
|
|
||||||
|
|
||||||
<!-- SELECTORS -->
|
|
||||||
<div class="column">
|
|
||||||
<h4 class="title is-4">Selectors</h4>
|
|
||||||
<div id="compare-widgets">
|
|
||||||
{{ embed(roots.test_filter) }}
|
|
||||||
{{ embed(roots.select_test) }}
|
|
||||||
<br>
|
|
||||||
{{ embed(roots.select_var) }}
|
|
||||||
<br>
|
|
||||||
{{ embed(roots.select_backend) }}
|
|
||||||
<br>
|
|
||||||
{{ embed(roots.outliers_filtering_compare) }}
|
|
||||||
<br>
|
|
||||||
<br>
|
|
||||||
{{ embed(roots.select_n_runs) }}
|
|
||||||
|
|
||||||
|
|
||||||
<br>
|
|
||||||
<br>
|
|
||||||
|
|
||||||
<b>Tip :</b> You can click on any element of the plots
|
|
||||||
to inspect the corresponding run in details.
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<div class="is-divider-vertical"></div>
|
|
||||||
|
|
||||||
|
|
||||||
<!-- PLOTS -->
|
|
||||||
<div class="column is-9">
|
|
||||||
<h3 class="title is-3">Plots</h3>
|
|
||||||
<div class="container">
|
|
||||||
<div class="card plot-card">
|
|
||||||
{{ embed(roots.s_tabs) }}
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<br>
|
|
||||||
|
|
||||||
<div class="card plot-card">
|
|
||||||
{{ embed(roots.sigma_plot) }}
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<br>
|
|
||||||
|
|
||||||
<div class="card plot-card">
|
|
||||||
{{ embed(roots.boxplot) }}
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
</div>
|
|
||||||
</main>
|
|
||||||
|
|
||||||
|
|
||||||
<!-- CONTENT : INSPECT RUNS -->
|
|
||||||
<main class="container" id="inspect-runs-container">
|
|
||||||
<div class="columns">
|
|
||||||
|
|
||||||
<!-- SELECTORS -->
|
|
||||||
<div class="column">
|
|
||||||
<h4 class="title is-4">Selectors</h4>
|
|
||||||
{{ embed(roots.select_run) }}
|
|
||||||
|
|
||||||
<br>
|
|
||||||
|
|
||||||
Group by :
|
|
||||||
{{ embed(roots.groupby_radio) }}
|
|
||||||
|
|
||||||
<br>
|
|
||||||
|
|
||||||
Filter by :
|
|
||||||
{{ embed(roots.filterby_radio) }}
|
|
||||||
{{ embed(roots.select_filter) }}
|
|
||||||
|
|
||||||
<br>
|
|
||||||
|
|
||||||
{{ embed(roots.outliers_filtering_inspect) }}
|
|
||||||
|
|
||||||
<br>
|
|
||||||
<br>
|
|
||||||
|
|
||||||
<h4 class="title is-4">Run metadata</h4>
|
|
||||||
|
|
||||||
<b>Date :</b>
|
|
||||||
<div id="run-date" style="display: inline;">
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<br>
|
|
||||||
|
|
||||||
<div id="is-git-commit">
|
|
||||||
|
|
||||||
<b>Hash :</b>
|
|
||||||
<div
|
|
||||||
id="run-hash" style="display: inline;">
|
|
||||||
</div>
|
|
||||||
<br>
|
|
||||||
<b>Author :</b>
|
|
||||||
<div id="run-author" style="display: inline;">
|
|
||||||
</div>
|
|
||||||
<br>
|
|
||||||
<b>Message :</b>
|
|
||||||
<div id="run-message" style="display: inline;">
|
|
||||||
</div>
|
|
||||||
|
|
||||||
{% if git_repo_linked %}
|
|
||||||
|
|
||||||
<br>
|
|
||||||
<br>
|
|
||||||
|
|
||||||
|
|
||||||
<a
|
|
||||||
id="git-commit-link"
|
|
||||||
href=""
|
|
||||||
target="_blank"
|
|
||||||
>
|
|
||||||
View this commit on {{git_host}}
|
|
||||||
</a>
|
|
||||||
|
|
||||||
{% endif %}
|
|
||||||
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<div id="not-git-commit">
|
|
||||||
|
|
||||||
This run is not linked to a Git commit.
|
|
||||||
|
|
||||||
</div>
|
|
||||||
|
|
||||||
</div>
|
|
||||||
|
|
||||||
|
|
||||||
<!-- PLOTS -->
|
|
||||||
<div class="column is-9">
|
|
||||||
<h3 class="title is-3">Plots</h3>
|
|
||||||
<div class="card plot-card" style="z-index: 3;">
|
|
||||||
{{ embed(roots.s_tabs_inspect) }}
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<br>
|
|
||||||
|
|
||||||
<div class="card plot-card" style="z-index: 2;">
|
|
||||||
{{ embed(roots.sigma_inspect) }}
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<br>
|
|
||||||
|
|
||||||
<div class="card plot-card" style="z-index: 1;">
|
|
||||||
{{ embed(roots.mu_inspect) }}
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
</div>
|
|
||||||
</main>
|
|
||||||
|
|
||||||
</div>
|
|
||||||
|
|
||||||
|
|
||||||
<!--- LOADER -->
|
|
||||||
<div id="loader">
|
|
||||||
|
|
||||||
{% if has_logo %}
|
|
||||||
<img id="loading-logo" src="{{logo_url}}">
|
|
||||||
{% else %}
|
|
||||||
<img id="loading-logo" src="https://avatars1.githubusercontent.com/u/12033642">
|
|
||||||
{% endif %}
|
|
||||||
</div>
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
<!-- JAVASCRIPT -->
|
|
||||||
<script>
|
|
||||||
|
|
||||||
// Listen to clicks on breadcrumb (for responsive header)
|
|
||||||
document.getElementById("navbar-burger")
|
|
||||||
.addEventListener("click", () => {
|
|
||||||
|
|
||||||
document.getElementById("navbar-burger")
|
|
||||||
.classList.toggle("is-active");
|
|
||||||
|
|
||||||
document.getElementById("navbar-content")
|
|
||||||
.classList.toggle("is-active");
|
|
||||||
|
|
||||||
})
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
// Helper function to navigate between views
|
|
||||||
function changeView(classPrefix) {
|
|
||||||
|
|
||||||
// Enable/disable the active class on buttons
|
|
||||||
let buttons = document.getElementById("buttons-container")
|
|
||||||
.childNodes;
|
|
||||||
let toggledButtonId = classPrefix + "-button";
|
|
||||||
|
|
||||||
for(let i=0; i<buttons.length; i++) {
|
|
||||||
console.log()
|
|
||||||
if(toggledButtonId == buttons[i].id) {
|
|
||||||
buttons[i].classList.add("is-active");
|
|
||||||
}
|
|
||||||
else if(buttons[i].classList != undefined) {
|
|
||||||
buttons[i].classList.remove("is-active");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Show hide the containers
|
|
||||||
let containers = document.getElementsByTagName("MAIN");
|
|
||||||
let toggledContainerId = classPrefix + "-container"
|
|
||||||
|
|
||||||
for(let i=0; i<containers.length; i++) {
|
|
||||||
if(toggledContainerId == containers[i].id) {
|
|
||||||
containers[i].style.display = "block";
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
containers[i].style.display = "none";
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Listen to clicks on "Compare runs" button
|
|
||||||
document.getElementById("compare-runs-button")
|
|
||||||
.addEventListener("click", () => {
|
|
||||||
// Nothing else to do for this button
|
|
||||||
changeView("compare-runs");
|
|
||||||
});
|
|
||||||
|
|
||||||
// Listen to clicks on "Inspect runs" button
|
|
||||||
// (dedicated function as this needs to be called in a CustomJS callback)
|
|
||||||
function goToInspectRuns() {
|
|
||||||
window.scrollTo(0, 0);
|
|
||||||
|
|
||||||
changeView("inspect-runs");
|
|
||||||
}
|
|
||||||
|
|
||||||
document.getElementById("inspect-runs-button")
|
|
||||||
.addEventListener("click", goToInspectRuns);
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
// Toggle the display properties of the loader/report
|
|
||||||
function removeLoader() {
|
|
||||||
document.getElementById("loader")
|
|
||||||
.style.display = "none";
|
|
||||||
|
|
||||||
document.getElementById("report")
|
|
||||||
.style.display = "";
|
|
||||||
}
|
|
||||||
|
|
||||||
// To detect the end of Bokeh initialization and remove the loader,
|
|
||||||
// we look at the number of children of a div containing widgets
|
|
||||||
let nChildren = document.getElementById('compare-widgets')
|
|
||||||
.getElementsByTagName('*').length;
|
|
||||||
|
|
||||||
function pollBokehLoading() {
|
|
||||||
let newNChildren = document.getElementById('compare-widgets')
|
|
||||||
.getElementsByTagName('*').length;
|
|
||||||
|
|
||||||
if(newNChildren != nChildren) {
|
|
||||||
removeLoader();
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
setTimeout(pollBokehLoading, 100);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
setTimeout(pollBokehLoading, 100);
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
// Update the run metadata (in inspect run mode)
|
|
||||||
function updateRunMetadata(runId) {
|
|
||||||
|
|
||||||
// Assume runId is the run's timestamp
|
|
||||||
let run = metadata[runId];
|
|
||||||
|
|
||||||
// If it is undefined, perform a search by name
|
|
||||||
// (by iterating metadata)
|
|
||||||
if(!run) {
|
|
||||||
for(let [key, value] of Object.entries(metadata)) {
|
|
||||||
|
|
||||||
if (!metadata.hasOwnProperty(key)) continue;
|
|
||||||
if(value.name == runId) {
|
|
||||||
run = value;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
document.getElementById("run-date").innerHTML = run.date;
|
|
||||||
|
|
||||||
if(run.is_git_commit) {
|
|
||||||
document.getElementById("is-git-commit").style.display = "";
|
|
||||||
document.getElementById("not-git-commit").style.display = "none";
|
|
||||||
|
|
||||||
document.getElementById("run-hash").innerHTML = run.hash;
|
|
||||||
document.getElementById("run-author").innerHTML = run.author;
|
|
||||||
document.getElementById("run-message").innerHTML = run.message;
|
|
||||||
|
|
||||||
{% if git_repo_linked %}
|
|
||||||
document.getElementById("git-commit-link")
|
|
||||||
.setAttribute("href", "{{commit_link}}" + run.hash);
|
|
||||||
{% endif %}
|
|
||||||
|
|
||||||
} else {
|
|
||||||
document.getElementById("is-git-commit").style.display = "none";
|
|
||||||
document.getElementById("not-git-commit").style.display = "";
|
|
||||||
|
|
||||||
document.getElementById("run-hash").innerHTML = "";
|
|
||||||
document.getElementById("run-author").innerHTML = "";
|
|
||||||
document.getElementById("run-message").innerHTML = "";
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
//Object containing metadata from all runs
|
|
||||||
metadata = {{metadata}}
|
|
||||||
|
|
||||||
// Initial run using the template arg
|
|
||||||
updateRunMetadata({{initial_timestamp}});
|
|
||||||
|
|
||||||
</script>
|
|
||||||
|
|
||||||
{% endblock %}
|
|
||||||
</body>
|
|
||||||
|
|
||||||
</html>
|
|
||||||
@ -1,7 +0,0 @@
|
|||||||
## Verificarlo CI : `{{dev_branch}}`
|
|
||||||
|
|
||||||
You are on the `{{ci_branch}}` branch, which is automatically updated with the
|
|
||||||
[Verificarlo](https://github.com/verificarlo/verificarlo) test results from
|
|
||||||
`{{dev_branch}}` (in the `vfcruns` directory).
|
|
||||||
|
|
||||||
You can start a Verificarlo CI server at anytime using the run files of this branch.
|
|
||||||
@ -1,42 +0,0 @@
|
|||||||
# This workflow will be executed when {{dev_branch}} is updated:
|
|
||||||
# it will run the configured tests and upload the results on vfc_ci_master.
|
|
||||||
|
|
||||||
image: verificarlo/verificarlo
|
|
||||||
|
|
||||||
|
|
||||||
stages:
|
|
||||||
- run_verificarlo_tests
|
|
||||||
|
|
||||||
|
|
||||||
run_verificarlo_tests:
|
|
||||||
stage: run_verificarlo_tests
|
|
||||||
|
|
||||||
before_script:
|
|
||||||
- git remote set-url origin https://{{username}}:${CI_PUSH_TOKEN}@{{remote_url}}.git
|
|
||||||
- git config --global user.email "{{email}}"
|
|
||||||
- git config --global user.name "{{username}}"
|
|
||||||
|
|
||||||
script:
|
|
||||||
# We will probably drop these installations when integrating CI into
|
|
||||||
# Verificarlo
|
|
||||||
- pip install numpy scipy pandas bokeh jinja2 tables GitPython
|
|
||||||
- apt update
|
|
||||||
- apt install wget
|
|
||||||
- wget https://raw.githubusercontent.com/verificarlo/significantdigits/main/sigdigits.py -P /usr/local/lib/python3.8/dist-packages
|
|
||||||
|
|
||||||
- ./vfc_ci test -g -r
|
|
||||||
- git_hash=$(git rev-parse --short "$CI_COMMIT_SHA")
|
|
||||||
- git fetch --all
|
|
||||||
- git checkout -b {{ci_branch}} origin/{{ci_branch}}
|
|
||||||
- mkdir -p vfcruns
|
|
||||||
- mv *.vfcrun.h5 vfcruns
|
|
||||||
- git add vfcruns/*
|
|
||||||
- git commit -m "[auto] New test results for commit ${git_hash}"
|
|
||||||
- git push
|
|
||||||
|
|
||||||
rules:
|
|
||||||
- if: '$CI_COMMIT_BRANCH == "{{dev_branch}}"'
|
|
||||||
|
|
||||||
artifacts:
|
|
||||||
paths:
|
|
||||||
- "*.vfcraw.h5"
|
|
||||||
@ -1,57 +0,0 @@
|
|||||||
# This workflow will be executed when {{dev_branch}} is updated:
|
|
||||||
# it will run the configured tests and upload the results on {{ci_branch}}.
|
|
||||||
|
|
||||||
name: "Verificarlo CI ({{dev_branch}})"
|
|
||||||
|
|
||||||
on:
|
|
||||||
# Triggers the workflow when {{dev_branch}} is updated
|
|
||||||
push:
|
|
||||||
branches: [ {{dev_branch}} ]
|
|
||||||
|
|
||||||
workflow_dispatch:
|
|
||||||
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
run_verificarlo_tests:
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
container: verificarlo/verificarlo
|
|
||||||
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v2
|
|
||||||
with:
|
|
||||||
fetch-depth: 0
|
|
||||||
|
|
||||||
# We will probably drop these installations when integrating CI into
|
|
||||||
# Verificarlo
|
|
||||||
- name: Install Python requirements
|
|
||||||
run: |
|
|
||||||
pip install numpy scipy pandas bokeh jinja2 tables GitPython
|
|
||||||
apt update
|
|
||||||
apt install wget
|
|
||||||
wget https://raw.githubusercontent.com/verificarlo/significantdigits/main/sigdigits.py -P /usr/local/lib/python3.8/dist-packages
|
|
||||||
|
|
||||||
- name: Run tests
|
|
||||||
# We assume the script is included in the repo for now
|
|
||||||
# (we'll probably want to remove "./" if the script ends up being integrated
|
|
||||||
# in Verificarlo and becomes available system-wide)
|
|
||||||
run: ./vfc_ci test -g -r
|
|
||||||
|
|
||||||
- name: Commit test results
|
|
||||||
run: |
|
|
||||||
git_hash=$(git rev-parse --short "$GITHUB_SHA")
|
|
||||||
|
|
||||||
git config --local user.email "action@github.com"
|
|
||||||
git config --local user.name "GitHub Action"
|
|
||||||
|
|
||||||
git checkout {{ci_branch}}
|
|
||||||
mkdir -p vfcruns
|
|
||||||
mv *.vfcrun.h5 vfcruns
|
|
||||||
git add vfcruns/*
|
|
||||||
git commit -m "[auto] New test results for commit ${git_hash}"
|
|
||||||
git push
|
|
||||||
|
|
||||||
- name: Upload raw results as artifacts
|
|
||||||
uses: actions/upload-artifact@v2
|
|
||||||
with:
|
|
||||||
{% raw %}name: ${{github.sha}}.vfcraw{% endraw %}
|
|
||||||
path: ./*.vfcraw.h5
|
|
||||||
@ -150,26 +150,33 @@ int test_cycle(H5File file, int cycle, std::string version,
|
|||||||
}
|
}
|
||||||
|
|
||||||
int main(int argc, char **argv) {
|
int main(int argc, char **argv) {
|
||||||
if (argc != 3) {
|
if (argc != 2) {
|
||||||
std::cerr << "Execute from within '/'" << std::endl;
|
std::cerr << "Execute from within '/'" << std::endl;
|
||||||
std::cerr << "usage: test_h5 <version> <path to cycles file>" << std::endl;
|
std::cerr << "usage: vfc_test_h5 <path to cycles file>" << std::endl;
|
||||||
return 1;
|
return 1;
|
||||||
}
|
}
|
||||||
std::string version(argv[1]);
|
std::vector<int> cycles_list = get_cycles_list(argv[1]);
|
||||||
std::vector<int> cycles_list = get_cycles_list(argv[2]);
|
|
||||||
H5File file(FILE_NAME, H5F_ACC_RDONLY);
|
H5File file(FILE_NAME, H5F_ACC_RDONLY);
|
||||||
|
|
||||||
vfc_probes probes = vfc_init_probes();
|
vfc_probes probes = vfc_init_probes();
|
||||||
probes = vfc_init_probes();
|
probes = vfc_init_probes();
|
||||||
|
|
||||||
|
std::vector<std::string> versions = {"maponia3", "sm1", "sm2", "sm3", "sm4"};
|
||||||
|
|
||||||
bool ok;
|
bool ok;
|
||||||
for (int i = 0; i < cycles_list.size(); i++) {
|
for (int i = 0; i < cycles_list.size(); i++) {
|
||||||
ok = test_cycle(file, cycles_list[i], version, &probes);
|
|
||||||
if (ok) {
|
for(const auto& version: versions) {
|
||||||
std::cout << "ok -- cycle " << std::to_string(i) << std::endl;
|
|
||||||
} else {
|
ok = test_cycle(file, cycles_list[i], version, &probes);
|
||||||
std::cerr << "failed -- cycle " << std::to_string(i) << std::endl;
|
if (ok) {
|
||||||
|
std::cout << "ok -- cycle " << std::to_string(i) << std::endl;
|
||||||
|
} else {
|
||||||
|
std::cerr << "failed -- cycle " << std::to_string(i) << std::endl;
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
vfc_dump_probes(&probes);
|
vfc_dump_probes(&probes);
|
||||||
|
|||||||
199
vfc_ci
199
vfc_ci
@ -1,199 +0,0 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
|
|
||||||
# This is the entry point of the Verificarlo CI command line interface, which is
|
|
||||||
# based on argparse and this article :
|
|
||||||
# https://mike.depalatis.net/blog/simplifying-argparse.html
|
|
||||||
# From here, 3 subcommands can be called :
|
|
||||||
# - setup : create a vfc_ci branch and workflow on the current Git repo
|
|
||||||
# - test : run and export test results according to the vfc_tests_config.json
|
|
||||||
# - serve : launch a Bokeh server to visualize run results
|
|
||||||
|
|
||||||
import argparse
|
|
||||||
|
|
||||||
##########################################################################
|
|
||||||
|
|
||||||
# Parameters validation helpers
|
|
||||||
|
|
||||||
|
|
||||||
def is_port(string):
|
|
||||||
value = int(string)
|
|
||||||
if value < 0 or value > 65535:
|
|
||||||
raise argparse.ArgumentTypeError("Value has to be between 0 and 65535")
|
|
||||||
return value
|
|
||||||
|
|
||||||
|
|
||||||
def is_directory(string):
|
|
||||||
import os
|
|
||||||
|
|
||||||
isdir = os.path.isdir(string)
|
|
||||||
if not isdir:
|
|
||||||
raise argparse.ArgumentTypeError("Directory does not exist")
|
|
||||||
|
|
||||||
return string
|
|
||||||
|
|
||||||
|
|
||||||
##########################################################################
|
|
||||||
|
|
||||||
# Subcommand decorator
|
|
||||||
|
|
||||||
cli = argparse.ArgumentParser(
|
|
||||||
description="Define, run, automatize, and visualize custom Verificarlo tests."
|
|
||||||
)
|
|
||||||
subparsers = cli.add_subparsers(dest="subcommand")
|
|
||||||
|
|
||||||
|
|
||||||
def subcommand(description="", args=[], parent=subparsers):
|
|
||||||
def decorator(func):
|
|
||||||
parser = parent.add_parser(func.__name__, description=description)
|
|
||||||
for arg in args:
|
|
||||||
parser.add_argument(*arg[0], **arg[1])
|
|
||||||
parser.set_defaults(func=func)
|
|
||||||
return decorator
|
|
||||||
|
|
||||||
|
|
||||||
def argument(*name_or_flags, **kwargs):
|
|
||||||
return ([*name_or_flags], kwargs)
|
|
||||||
|
|
||||||
|
|
||||||
##########################################################################
|
|
||||||
|
|
||||||
# "setup" subcommand
|
|
||||||
|
|
||||||
@subcommand(
|
|
||||||
description="Create an automated workflow to execute Verificarlo tests.",
|
|
||||||
args=[
|
|
||||||
argument(
|
|
||||||
"git_host",
|
|
||||||
help="""
|
|
||||||
specify where your repository is hosted
|
|
||||||
""",
|
|
||||||
choices=["github", "gitlab"]
|
|
||||||
)
|
|
||||||
]
|
|
||||||
)
|
|
||||||
def setup(args):
|
|
||||||
import ci.setup
|
|
||||||
ci.setup.setup(args.git_host)
|
|
||||||
|
|
||||||
# "test" subcommand
|
|
||||||
|
|
||||||
|
|
||||||
@subcommand(
|
|
||||||
description="Execute predefined Verificarlo tests and save their results.",
|
|
||||||
args=[
|
|
||||||
argument(
|
|
||||||
"-g", "--is-git-commit",
|
|
||||||
help="""
|
|
||||||
When specified, the last Git commit of the local repository (working
|
|
||||||
directory) will be fetched and associated with the run.
|
|
||||||
""",
|
|
||||||
action="store_true"
|
|
||||||
),
|
|
||||||
argument(
|
|
||||||
"-r", "--export-raw-results",
|
|
||||||
help="""
|
|
||||||
Specify if an additional HDF5 file containing the raw results must be
|
|
||||||
exported.
|
|
||||||
""",
|
|
||||||
action="store_true"
|
|
||||||
),
|
|
||||||
argument(
|
|
||||||
"-d", "--dry-run",
|
|
||||||
help="""
|
|
||||||
Perform a dry run by not saving the test results.
|
|
||||||
""",
|
|
||||||
action="store_true"
|
|
||||||
)
|
|
||||||
]
|
|
||||||
)
|
|
||||||
def test(args):
|
|
||||||
import ci.test
|
|
||||||
ci.test.run(args.is_git_commit, args.export_raw_results, args.dry_run)
|
|
||||||
|
|
||||||
# "serve" subcommand
|
|
||||||
|
|
||||||
|
|
||||||
@subcommand(
|
|
||||||
description="""
|
|
||||||
Start a server to visualize Verificarlo test results.
|
|
||||||
""",
|
|
||||||
args=[
|
|
||||||
argument(
|
|
||||||
"-s", "--show",
|
|
||||||
help="""
|
|
||||||
Specify if the report must be opened in the browser at server
|
|
||||||
startup.
|
|
||||||
""",
|
|
||||||
action="store_true"
|
|
||||||
),
|
|
||||||
argument(
|
|
||||||
"-gd", "--git-directory",
|
|
||||||
help="""
|
|
||||||
Path to a local Git repository. The report will be linked to the
|
|
||||||
remote URL (GitHub and GitLab are supported).
|
|
||||||
""",
|
|
||||||
type=is_directory
|
|
||||||
),
|
|
||||||
argument(
|
|
||||||
"-gu", "--git-url",
|
|
||||||
help="""
|
|
||||||
GitHub or GitLab repository URL. The report will be linked to this
|
|
||||||
URL.
|
|
||||||
""",
|
|
||||||
type=str
|
|
||||||
),
|
|
||||||
argument(
|
|
||||||
"-p", "--port",
|
|
||||||
help="""
|
|
||||||
The port on which the server will run. Defaults to 8080.
|
|
||||||
""",
|
|
||||||
type=is_port,
|
|
||||||
default=8080
|
|
||||||
),
|
|
||||||
argument(
|
|
||||||
"-a", "--allow-origin",
|
|
||||||
help="""
|
|
||||||
The origin (URL) from which the report will be accessible.
|
|
||||||
Port number must not be specified. Defaults to * (allow everything).
|
|
||||||
""",
|
|
||||||
type=str,
|
|
||||||
default="*"
|
|
||||||
),
|
|
||||||
argument(
|
|
||||||
"-l", "--logo",
|
|
||||||
help="""
|
|
||||||
Specify the URL of an image to be displayed in the report header.
|
|
||||||
""",
|
|
||||||
type=str
|
|
||||||
)
|
|
||||||
]
|
|
||||||
)
|
|
||||||
def serve(args):
|
|
||||||
|
|
||||||
# git_directory and git_url are supposed to be exclusive
|
|
||||||
if args.git_directory is not None and args.git_url is not None:
|
|
||||||
raise argparse.ArgumentTypeError(
|
|
||||||
"\"-gd\" / \"--git-directory\" and \"-gu\" / \"--git-url\" are "
|
|
||||||
"mutually exclusive. Please make sure to use at most one of them."
|
|
||||||
)
|
|
||||||
|
|
||||||
import ci.serve
|
|
||||||
ci.serve.serve(
|
|
||||||
args.show,
|
|
||||||
args.git_directory,
|
|
||||||
args.git_url,
|
|
||||||
args.port,
|
|
||||||
args.allow_origin,
|
|
||||||
args.logo
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
###############################################################################
|
|
||||||
|
|
||||||
# Main command group and entry point
|
|
||||||
if __name__ == "__main__":
|
|
||||||
args = cli.parse_args()
|
|
||||||
if args.subcommand is None:
|
|
||||||
cli.print_help()
|
|
||||||
else:
|
|
||||||
args.func(args)
|
|
||||||
@ -1,69 +1,10 @@
|
|||||||
|
|
||||||
{
|
{
|
||||||
"make_command": "make -f Makefile.verificarlo",
|
"make_command": "make -f Makefile.verificarlo",
|
||||||
"executables": [
|
"executables": [
|
||||||
{
|
{
|
||||||
"executable": "bin/vfc_test_h5",
|
"executable": "bin/vfc_test_h5",
|
||||||
"parameters" : "maponia3 vfc_ci_cycles.txt",
|
"parameters" : "vfc_ci_cycles.txt",
|
||||||
"vfc_backends": [
|
|
||||||
{
|
|
||||||
"name": "libinterflop_mca.so",
|
|
||||||
"repetitions": 50
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "libinterflop_mca.so --mode=rr",
|
|
||||||
"repetitions": 50
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
|
|
||||||
{
|
|
||||||
"executable": "bin/vfc_test_h5",
|
|
||||||
"parameters" : "sm1 vfc_ci_cycles.txt",
|
|
||||||
"vfc_backends": [
|
|
||||||
{
|
|
||||||
"name": "libinterflop_mca.so",
|
|
||||||
"repetitions": 50
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "libinterflop_mca.so --mode=rr",
|
|
||||||
"repetitions": 50
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
|
|
||||||
{
|
|
||||||
"executable": "bin/vfc_test_h5",
|
|
||||||
"parameters" : "sm2 vfc_ci_cycles.txt",
|
|
||||||
"vfc_backends": [
|
|
||||||
{
|
|
||||||
"name": "libinterflop_mca.so",
|
|
||||||
"repetitions": 50
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "libinterflop_mca.so --mode=rr",
|
|
||||||
"repetitions": 50
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
|
|
||||||
{
|
|
||||||
"executable": "bin/vfc_test_h5",
|
|
||||||
"parameters" : "sm3 vfc_ci_cycles.txt",
|
|
||||||
"vfc_backends": [
|
|
||||||
{
|
|
||||||
"name": "libinterflop_mca.so",
|
|
||||||
"repetitions": 50
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "libinterflop_mca.so --mode=rr",
|
|
||||||
"repetitions": 50
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
|
|
||||||
{
|
|
||||||
"executable": "bin/vfc_test_h5",
|
|
||||||
"parameters" : "sm4 vfc_ci_cycles.txt",
|
|
||||||
"vfc_backends": [
|
"vfc_backends": [
|
||||||
{
|
{
|
||||||
"name": "libinterflop_mca.so",
|
"name": "libinterflop_mca.so",
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user