From 1d1da0093143327e49a894d3fa360d38eaca36a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Delval?= Date: Thu, 29 Apr 2021 23:41:23 +0200 Subject: [PATCH] Integration of vfc_ci The vfc_ci tool has been directly added to the repository, since it's not integrated into Verificarlo yet. The vfc_test_h5.cpp file defines a test inspired by test_h5.cpp that reads a list of cycles and dump the vfc_probes for these cycles. --- .gitignore | 2 + Makefile.vfc_ci | 64 ++ ci/__init__.py | 0 ci/__pycache__/__init__.cpython-39.pyc | Bin 0 -> 154 bytes ci/__pycache__/serve.cpython-39.pyc | Bin 0 -> 656 bytes ci/__pycache__/setup.cpython-39.pyc | Bin 0 -> 3296 bytes ci/__pycache__/test.cpython-39.pyc | Bin 0 -> 7108 bytes ci/serve.py | 28 + ci/setup.py | 155 +++++ ci/test.py | 387 ++++++++++++ .../__pycache__/compare_runs.cpython-39.pyc | Bin 0 -> 9285 bytes .../compare_variables.cpython-39.pyc | Bin 0 -> 463 bytes .../__pycache__/helper.cpython-39.pyc | Bin 0 -> 2839 bytes .../__pycache__/inspect_runs.cpython-39.pyc | Bin 0 -> 10125 bytes .../__pycache__/plot.cpython-39.pyc | Bin 0 -> 2455 bytes ci/vfc_ci_report/compare_runs.py | 556 +++++++++++++++++ ci/vfc_ci_report/helper.py | 166 +++++ ci/vfc_ci_report/inspect_runs.py | 588 ++++++++++++++++++ ci/vfc_ci_report/main.py | 217 +++++++ ci/vfc_ci_report/plot.py | 151 +++++ ci/vfc_ci_report/templates/index.html | 480 ++++++++++++++ ci/workflow_templates/ci_README.j2.md | 7 + ci/workflow_templates/gitlab-ci.j2.yml | 42 ++ .../vfc_test_workflow.j2.yml | 57 ++ include/vfc_hashmap.h | 248 ++++++++ include/vfc_probe.h | 254 ++++++++ tests/vfc_test_h5.cpp | 196 ++++++ vfc_ci | 205 ++++++ vfc_ci_cycles.txt | 2 + vfc_tests_config.json | 92 +++ 30 files changed, 3897 insertions(+) create mode 100644 Makefile.vfc_ci create mode 100644 ci/__init__.py create mode 100644 ci/__pycache__/__init__.cpython-39.pyc create mode 100644 ci/__pycache__/serve.cpython-39.pyc create mode 100644 ci/__pycache__/setup.cpython-39.pyc create mode 100644 ci/__pycache__/test.cpython-39.pyc create mode 100755 ci/serve.py create mode 100644 ci/setup.py create mode 100755 ci/test.py create mode 100644 ci/vfc_ci_report/__pycache__/compare_runs.cpython-39.pyc create mode 100644 ci/vfc_ci_report/__pycache__/compare_variables.cpython-39.pyc create mode 100644 ci/vfc_ci_report/__pycache__/helper.cpython-39.pyc create mode 100644 ci/vfc_ci_report/__pycache__/inspect_runs.cpython-39.pyc create mode 100644 ci/vfc_ci_report/__pycache__/plot.cpython-39.pyc create mode 100644 ci/vfc_ci_report/compare_runs.py create mode 100644 ci/vfc_ci_report/helper.py create mode 100644 ci/vfc_ci_report/inspect_runs.py create mode 100644 ci/vfc_ci_report/main.py create mode 100644 ci/vfc_ci_report/plot.py create mode 100644 ci/vfc_ci_report/templates/index.html create mode 100644 ci/workflow_templates/ci_README.j2.md create mode 100644 ci/workflow_templates/gitlab-ci.j2.yml create mode 100644 ci/workflow_templates/vfc_test_workflow.j2.yml create mode 100644 include/vfc_hashmap.h create mode 100644 include/vfc_probe.h create mode 100644 tests/vfc_test_h5.cpp create mode 100755 vfc_ci create mode 100644 vfc_ci_cycles.txt create mode 100644 vfc_tests_config.json diff --git a/.gitignore b/.gitignore index 9980d03..a295ba0 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,5 @@ Slater* Updates* datasets/dataset.* bin/ +*.vfcrun.hd5 +*.vfcraw.hd5 diff --git a/Makefile.vfc_ci b/Makefile.vfc_ci new file mode 100644 index 0000000..8064b40 --- /dev/null +++ b/Makefile.vfc_ci @@ -0,0 +1,64 @@ +## Compiler +CXX = verificarlo-c++ + +## Compiler flags +H5FLAGS = -I/usr/include/hdf5/serial -lhdf5_serial -lhdf5_cpp +# H5FLAGS = -lhdf5 -lhdf5_cpp +CXXFLAGS = -O0 -g $(H5FLAGS) + +INCLUDE = -I $(INC_DIR)/ -I="/usr/include" +DEPS_CXX = $(OBJ_DIR)/SM_MaponiA3.o $(OBJ_DIR)/SM_MaponiA3S.o $(OBJ_DIR)/SM_Standard.o $(OBJ_DIR)/SM_Helpers.o + +SRC_DIR := src +TST_DIR := tests +INC_DIR := include +OBJ_DIR := build +BIN_DIR := bin + +EXEC := $(BIN_DIR)/vfc_test_h5 + +## Build tagets +.PHONY: all clean distclean + +all: $(EXEC) + +clean: + @rm -vrf $(OBJ_DIR) + @rm -vrf $(BIN_DIR) + +distclean: clean + @rm -vrf $(BIN_DIR) \ + Slater* Updates.dat + + +#### COMPILING +$(BIN_DIR) $(OBJ_DIR): + mkdir -p $@ + +### IMPLICIT BUILD RULES +## C++ objects +$(OBJ_DIR)/%.o: $(TST_DIR)/%.cpp $(INC_DIR)/* | $(OBJ_DIR) + $(CXX) $(CXXFLAGS) $(INCLUDE) -c -o $@ $< + +$(OBJ_DIR)/%.o: $(SRC_DIR)/%.cpp $(INC_DIR)/* | $(OBJ_DIR) + $(CXX) $(CXXFLAGS) -fPIE $(INCLUDE) -c -o $@ $< + +## HDF5/C++ objects +$(OBJ_DIR)/%_h5.o: $(TST_DIR)/%_h5.cpp $(INC_DIR)/* | $(OBJ_DIR) + $(CXX) $(H5CXXFLAGS) $(INCLUDE) -c -o $@ $< + +### EXPLICIT BUILD RULES +## special compiler flag -fPIC otherwise h5c++ builds fail +$(OBJ_DIR)/SM_MaponiA3.o: $(SRC_DIR)/SM_MaponiA3.cpp $(INC_DIR)/* | $(OBJ_DIR) + $(CXX) $(CXXFLAGS) -fPIC $(INCLUDE) -c -o $@ $< + +$(OBJ_DIR)/SM_MaponiA3S.o: $(SRC_DIR)/SM_MaponiA3S.cpp $(INC_DIR)/* | $(OBJ_DIR) + $(CXX) $(CXXFLAGS) -fPIC $(INCLUDE) -c -o $@ $< + +$(OBJ_DIR)/SM_Standard.o: $(SRC_DIR)/SM_Standard.cpp $(INC_DIR)/* | $(OBJ_DIR) + $(CXX) $(CXXFLAGS) -fPIC $(INCLUDE) -c -o $@ $< + + +#### LINKING +$(BIN_DIR)/vfc_test_h5: $(OBJ_DIR)/vfc_test_h5.o $(DEPS_CXX) | $(BIN_DIR) + $(CXX) $(CXXFLAGS) $(INCLUDE) -o $@ $^ $(H5FLAGS) diff --git a/ci/__init__.py b/ci/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ci/__pycache__/__init__.cpython-39.pyc b/ci/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0135b7144a5614d2edd969d33d163e71293481c8 GIT binary patch literal 154 zcmYe~<>g`kg3HOB2_X70h(HF6K#l_t7qb9~6oz01O-8?!3`HPe1o6vWKO;XkRX?$` zC^aWDHBa9qKe;qFHLs*tKRmxETR)&EKfNe1H#ajcUB4_XIX*d4ABf`PGxIV_;^XxS VDsOSvChpB8660Tn@>mxA!P}<*iA!|xU${rwQ@Hm{))Si zf5|HoGaCy$rzg~Bz32D6@7exhr<_bifJW}VfB8)TetK|jSc7-y`WXfeoV1Vwr#$!u zIpO3ZEZ8q(;JS+Z^k{z11pPg_HW)0Zgp;0fdc@Vo$Py0gK+Sky$#XzV2mFdNOFhP( z*oH&xiibADT5RAexu+(0AjgD9HnCut4RAy@UdGsQVc?$Giw8nbDJqMv4^f)xmE1Wv zOzXzXc%wvRq}tEs`Y`g9t!jN9x3ZQ#`E(=KVl~r3-3r9n?XoJWrch!dmBGR+P0O~G zyL2aRw5-;`q*66?)4!SP*8_dlsA}z(%=IF4RBFdIrCGTcRrF;iTv8OvrWHluBE8o} zbc>N=gWL&9EB=dhv4^M>H$W9`JSZqm=jMr*?%6!RHpe_vA>Mf&SF-EMp8o*#1o5BC zY$ZF9m0Km+M)cX0thSx#jm|zxwaz{$S*x<^n!bMg?5t|C;U->g_AVSU|H}FW1_;KK sk>oT^867i9rhi$WrpPa-3UC_o;rz4k6bYAHc9L&f@s=Ud{zpsbKOSDIjsO4v literal 0 HcmV?d00001 diff --git a/ci/__pycache__/setup.cpython-39.pyc b/ci/__pycache__/setup.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a6aeba4f88a4a1f4ee501b7334633af89fe7d7f4 GIT binary patch literal 3296 zcma)8TW=f372eq!mlQ?OwrnRw(JmY~4(o(Wf}ll>AZTE#jayp@Vy9^~4cOJrkX&lH zOUw+Vh*-T;MS(sD^=sd#NB@Qb{UQ6>r~ZW$sQaBEDJz13b_veq%$zf4&gDB3Z*Dd$ ze5ZSV`NKc1TGl^la`Ce<`33&!0UB;`mRZrVXAv`VJF?N+nKN-Ccj84Jv$(_EKUtB_ zJ?^8g@fr`%2b?{zI`w}+lC|5htBsxfAeBWv5xK6~zf3dnDsO z{O{whBpPjv*}^uhqun!h#*S_6X}`2j7-y&K*eRVwjoYBRr8^Evt7P0cwU0er=Pvh7 z?7#lt*f0G>qx8-gv%bWoT>MTPX65boEXCionE8E$mRnHV-+j(1KQqeBZygqLe~=Z2 zeJDK5Vl7lJN&BDgeEi_?PIvr%cfz0BRfCIz{+^8UWK`9$yc~X4Uu~Q4iXC<~D!WjX zJB{_Is_|6j@kCU$cuyHh(C-gmnSNiAJ*0!CY7T`yKezIw$YHO_Ev6!`yhE94(Wyx? znIxOZCh<5TKd6GWvhoVi?7`@{C2wN*-TmIEn226HlOjt+-g{6aGxCe-eTj?qK9j{z z#*;~!4|@lLq>sA{Q=P?oxO7hmJ)3r?M^$4e^1c)?pNJNODmvJ5wpoicoR-sM@^#D; zm#C|66aBeg-(|1#t84t~uKeni2)2D&J}W)=*MWcGMfjJMHh1CQ8uyL^#6^7>A!TsJ zxX){+HnVi240vEXUoY#6jk5Mi&;R+N=W$wn-+spA+vdCmou`|8lstGF6w5Y za(h>jK`S5r7|r(PcV7{hlDwnIBa^DwwSE z1f5Vdq?i<1^ohGg*1;X=A}~Vdirj!?lT(tSS5l38|r6UpB8{L_m_C2L)|aAD}TQ>jX9>t;=q(CVR_iAcX@K*jF)Y z+Ag!#zkmCI3!3bJFCn9^KT2y4e*@XSH>te&(XL z)&4cKn~QC}F*MxzI!QeN#27kYVM(XixWzZ=bkxWefe>gb>0>|ybg9t|c>8pzg*CoW zI{XTtfdK0!zec<#Si^XmdH~XO14zFEw{EzUXjSxgw79SoOB*u&6bS$sGNqvTE+#R`<*T5_X`bA^}Q9 z9=0c9{?WsHP+V}v$Cp)Jqb95gy@kj_Q%dP7u;=`5m-rt8Nx)~ITns|80bz~Sotelv0!h0^hjr?WIM0l@Jb-{}dwvF6R5R#Qkff>~iL%0^K zNRPTnaUjf#DGHL2NcIcV3~wsaTz3NbCTVqredaU! zKfc>!jTgSx_Dz|1VZU%)j9$2YgUZGW`7=oD)MXc4<)|ZNYHH+<(|jDif1!@jd(jxu ha9q&Q)G<@Sf)6JJpJn1B2YbznGq9l}Y1a;V{{`8XkHr80 literal 0 HcmV?d00001 diff --git a/ci/__pycache__/test.cpython-39.pyc b/ci/__pycache__/test.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..fa73d452612722f74b4fa60f0eb72f15333d46a3 GIT binary patch literal 7108 zcma)B&669)b)W7T3zxNpop28rucUq6ffW zff=xSKoTe*2d{le6%$n|m!w=asFV2@BnMZ{Ip#d4RHafCCdZuOIAMOT2LN}MQYF}$ z?&&xE@w(sZ_kQoKU#S!g{G9a9KKt8m8OA?T&x%{->t+~c}7 zJyX|~X9*ryjht@TUS8LZSI~9QE9tuIRfOqPh2^b?oUngpc&j2W9MmU7K@?H1iIOOz zJ}D}qih5nFh*i|5#0jy6dPAHP>!>%yDY1e2v|zhNZSy(!Gv+^Zpcq#ZW5y=NQ#Q5k8&S(L5t~V<`ZTcpBlRcM%bfdIl}tCM_e?L`~>|(ZlBA)!YcOBF%wJVw~U!198vfgIHUFz zV`6m+II3dE=zyMZ@I_xL^aTe{-5DqMeHn#O+x^kQRLpfS*M z*R0e^0+poBLtlpeUN1=PhkkDusI){~_x$F55Q!Q~^Ub(7>_=++{;Pa`N?c_Vg1KQ& zXqNW^R|dWS>(W(mKX8Z9eiT27h^pd9G7{itug*5#buS8gxd5A^=@ zo@o!*a?%OAj2q0ZGSL;WYfY8Kk;vY;~*{EeB2BMNf<||*$bjHhc3NQYRe$P=UOgxT9|F% zC!I7uSB%tolxeL>tHI--Ii%4Ifhp6x4AiifD1B;Ku?iN`m17%pK~&#CVQ|i>yv#V` zFD$FbEM~tbT1VUEmu+iL|wu?<*3Wu%EW}-u%=dYPJgd{$#+XwBKO3eTDaTx z78I*Vi+ zy)YSh#bz9}LJ>p|GtE<5(K?`yxZ@83xdmd%qX9%Fr9Vn}f0)_>t@vpXs@Rynb0t#y z`+8X~|9%h!((k2fDr`q#D{T5v(hy-gOjKI9vt-kGZ%^UsCqZgn-}#`HlT})*EJBsY za8C#-qN8!hl>?&8k^s4&)0-Gn?_>ImJUQXaWCEyQIe*kt=xu zQhjaxJ-m=y69##$;*IZ88#aF~XDi7alCwR_^Az@2P zEtg46p2ITo4Jzj6PMeT$xj1h1ww8kYI*~{M@(-!_1{L~1F3=loUv5)zk%~8|cnbxj zO$RC8rb2iBBYGnfbLg_X_Nr_ez4%ce8+&x4C3>sVC=6a?Hk~c6^2NE@zq9l5623L5 z!{SHENG4C8;s7t>?2}S7XA)QW<;xRL3$UpS7q>;%z&@0 zYhNpvF@-N1ZwPb38+d+-k)LAhl>aG@idlWpfX}pMkjGPU_SVk$6q#r;f~uiayIt?X zJf=*JTiQq+@`3j{p>oOYxTt0DU>HiW`>`tm)r0~DgR~N=2FwFw2LG0%RtLBPj${Zk zCeu86DZd?z&)*7?W``Ux44hA==8_+Ed%l8?)b1S{=yO zv$JKrO2%s-$VI`RLE?ITFBkOvu&38Zb7}yq3qwavIdvAB(DQlFmqF~6Xx8~L9M^{v z3)O2ZM*a%(C^GFv=Do=7{g+*^U<1qCVGb|ybud&}|#HmMB7FU13c*C?>w=wu~FbPtvJ{adw7Ig+upR6s9==t$1k2AA??m z<;Hfc-+M+zMMPgNnC8VwHX7~KO=FS+%@HT21+g}84UO<|6QIGH;zxK zTT<2D!P+-JXUhA~5Wjc8ehPdrFdvGHVk`$v|8I5Xs}oA`}={` z0$7~!T7O@J(yha^!3e2((jRCY_evQsB5w4qbd)Gg@xpQOpopY&0cx}wjxno}6{w;I08%trR20;=M zLQ~_Dhxyz-uj)XH<4v09&fVMBZtga2|M1?OAKrWC+#4>1vkq91R67t|0r%JMaT51Vgre5eqz$*eKC55UM>0U=vw~l*cNy)kq~lKg~%$Y6o(M zINPA&EEOlI8;nNOq$r>*VBMUX6;0JVf*U(?vPWoN z#$jIfTV3=cxY>ot6Kk5IlgH@mbNZ=c^o==vb3vccJOw@X=|i-lrfgjSe1Zu&v!fO6 zm2)QmM_Qy-Q7i)28BcJUyM`o30cg2IqMi+}J3@$1!UtKQ;;8z_6QETgLb-;Q@macn z6n-}0&2fRgB>W_>>g51T`~FumM@|d*7VV*6d(9x8jr?BkL_|9`Wqi0+aJoie?4jBO4 zLKS@F@tj^P@0seYi(XZ|eHeuQ{}-1{EGYvjzG|mmTQ0SNr4BEXnh4}-tc}t5{o_^S zWGr8{l~;T)MC1lL^Y*3X=YP8Tl6>jnlKiiS$bWnZGy0D`CsM$6uu`>>B;%%a$IBzJ zB!T9z<3waZ422T%PpD|2@SMf6vO_cqfND(Jy<-}dt{l^;GWMuk^f&m1xz5(vhPB2v zOgy=DAfa`3TK@qHh3IqScsU3;JOmL{f+E3XM>Nm&EjdUiiXmLY1t!bONr zfI%|UBoOmYU%%`KqM0k<0FRrCrjv;=0$eG@quClc%o0<#V*4 z$@f8q3)gWjc~SWzBJj+$eMW?%b|tN-PW-5`V04>gP8P<1+kumLVcBayza(wh1+__L zZSkYT*3!M4!N`}0_W+m{t+|{@_Q17>fezS@`7}4Rfi`tM7JYQYge*Xgq&~^bY#>34 zkl-v21V~61@}J}~2PAuFBV!X46DRwEne&t>n3X1$&Kn@Yn%N>px#kIzznO6Ln@O$< z94mliGxvjatR)xXVqmtW~klzDIPU76%F4V7Lu zSfF03Xh^CP1UmB5SWDjsZT!5#5@C8p!qjuLsqreIRJn=5D^P%q++8z4R2(-tqE&Nr z`0AaQ<40_@F;DF0!EghnNj7M_mIk3!eUq~kFNF`sud|5}H|Y{nSWWoS)}bpfqy=ON z177B*ttu%5&;gwtz`RTF_p2KMW;sCI6Yra2@9@;;d;h|xm_qAeZ4FC_5$u^9G(cgb8SO8pVc erURKD5VcH;Sq0~U^L6JnXT^El*>T=*HvSjT_R^97 literal 0 HcmV?d00001 diff --git a/ci/serve.py b/ci/serve.py new file mode 100755 index 0000000..8685fb0 --- /dev/null +++ b/ci/serve.py @@ -0,0 +1,28 @@ +# Server for the Verificarlo CI report. This is simply a wrapper to avoid +# calling Bokeh directly. + +import os + +def serve(show, git_directory, git_url, port, allow_origin, logo_url): + + # Prepare arguments + show = "--show" if show else "" + + git = "" + if git_directory != None: + git = "git directory %s" % git_directory + if git_url != None: + git = "git url %s" % git_url + + logo = "" + if logo_url != None: + logo = "logo %s" % logo_url + + dirname = os.path.dirname(__file__) + + + # Call the "bokeh serve" command on the system + command = "bokeh serve %s/vfc_ci_report %s --allow-websocket-origin=%s:%s --port %s --args %s %s" \ + % (dirname, show, allow_origin, port, port, git, logo) + + os.system(command) diff --git a/ci/setup.py b/ci/setup.py new file mode 100644 index 0000000..1159963 --- /dev/null +++ b/ci/setup.py @@ -0,0 +1,155 @@ +# Helper script to set up Verificarlo CI on the current branch + +import git + +import sys +import os +from jinja2 import Environment, FileSystemLoader + +################################################################################ + + + # Helper functions + +def gen_readme(dev_branch, ci_branch): + + # Init template loader + path = os.path.dirname(os.path.abspath(__file__)) + env = Environment(loader=FileSystemLoader(path)) + template = env.get_template("workflow_templates/ci_README.j2.md") + + # Render template + render = template.render(dev_branch=dev_branch, ci_branch=ci_branch) + + # Write template + with open("README.md", "w") as fh: + fh.write(render) + + + +def gen_workflow(git_host, dev_branch, ci_branch, repo): + + # Init template loader + path = os.path.dirname(os.path.abspath(__file__)) + env = Environment(loader=FileSystemLoader(path)) + + + if git_host == "github": + # Load template + template = env.get_template("workflow_templates/vfc_test_workflow.j2.yml") + + # Render it + render = template.render(dev_branch=dev_branch, ci_branch=ci_branch) + + # Write the file + filename = ".github/workflows/vfc_test_workflow.yml" + os.makedirs(os.path.dirname(filename), exist_ok=True) + with open(filename, "w") as fh: + fh.write(render) + + + if git_host == "gitlab": + template = env.get_template("workflow_templates/gitlab-ci.j2.yml") + + # Ask for the user who will run the jobs (Gitlab specific) + username = input("[vfc_ci] Enter the name of the user who will run the CI jobs:") + email = input("[vfc_ci] Enter the e-mail of the user who will run the CI jobs:") + + remote_url = repo.remotes[0].config_reader.get("url") + remote_url = remote_url.replace("http://", "") + remote_url = remote_url.replace("https://", "") + + render = template.render( + dev_branch=dev_branch, + ci_branch=ci_branch, + username=username, + email=email, + remote_url = remote_url + ) + + filename = ".gitlab-ci.yml" + with open(filename, "w") as fh: + fh.write(render) + + + +################################################################################ + +def setup(git_host): + + + # Init repo and make sure that the workflow setup is possible + + repo = git.Repo(".") + repo.remotes.origin.fetch() + + # Make sure that repository is clean + assert(not repo.is_dirty()), "Error [vfc_ci]: Unstaged changes detected " \ + "in your work tree." + + dev_branch = repo.active_branch + dev_branch_name = str(dev_branch) + dev_remote = dev_branch.tracking_branch() + + # Make sure that the active branch (on which to setup the workflow) has a remote + assert(dev_remote != None), "Error [vfc_ci]: The current branch doesn't " \ + "have a remote." + + # Make sure that we are not behind the remote (so we can push safely later) + rev = "%s...%s" % (dev_branch_name, str(dev_remote)) + commits_behind = list(repo.iter_commits(rev)) + assert(commits_behind == []), "Error [vfc_ci]: The local branch seems " \ + "to be at least one commit behind remote." + + + + # Commit the workflow on the current (dev) branch + + ci_branch_name = "vfc_ci_%s" % dev_branch_name + gen_workflow(git_host, dev_branch_name, ci_branch_name, repo) + repo.git.add(".") + repo.index.commit("[auto] Set up Verificarlo CI on this branch") + repo.remote(name="origin").push() + + + + # Create the CI branch (orphan branch with a readme on it) + # (see : https://github.com/gitpython-developers/GitPython/issues/615) + + repo.head.reference = git.Head(repo, "refs/heads/"+ ci_branch_name) + + repo.index.remove(["*"]) + gen_readme(dev_branch_name, ci_branch_name) + repo.index.add(["README.md"]) + + repo.index.commit( + "[auto] Create the Verificarlo CI branch for %s" % dev_branch_name, + parent_commits=None + ) + repo.remote(name="origin").push( + refspec="%s:%s" % (ci_branch_name, ci_branch_name) + ) + + # Force checkout back to the original (dev) branch + repo.git.checkout(dev_branch_name, force=True) + + + + # Print termination messages + + print( + "Info [vfc_ci]: A Verificarlo CI workflow has been setup on " \ + "%s." % dev_branch_name + ) + print( + "Info [vfc_ci]: Make sure that you have a \"vfc_tests_config.json\" on " \ + "this branch. You can also perform a \"vfc_ci test\" dry run before "\ + "pushing other commits." + ) + + if git_host == "gitlab": + print( + "Info [vfc_ci]: Since you are using GitLab, make sure that you " \ + "have created an access token for the user you specified (registered "\ + "as a variable called \"CI_PUSH_TOKEN\" in your repository)." + ) diff --git a/ci/test.py b/ci/test.py new file mode 100755 index 0000000..ce21010 --- /dev/null +++ b/ci/test.py @@ -0,0 +1,387 @@ +# This script reads the vfc_tests_config.json file and executes tests accordingly +# It will also generate a ... .vfcrun.hd5 file with the results of the run + +import os +import json + +import calendar +import time + +# Forcing an older pickle protocol allows backwards compatibility when reading +# HDF5 written in 3.8+ using an older version of Python +import pickle +pickle.HIGHEST_PROTOCOL = 4 + +import pandas as pd +import numpy as np +import scipy.stats + +import sigdigits as sd + +# Magic numbers +min_pvalue = 0.05 +max_zscore = 3 + + +################################################################################ + + + # Helper functions + +# Read a CSV file outputted by vfc_probe as a Pandas dataframe +def read_probes_csv(filepath, backend, warnings, execution_data): + + try: + results = pd.read_csv(filepath) + + except FileNotFoundError: + print( + "Warning [vfc_ci]: Probes not found, your code might have crashed " \ + "or you might have forgotten to call vfc_dump_probes" + ) + warnings.append(execution_data) + return pd.DataFrame( + columns = ["test", "variable", "values", "vfc_backend"] + ) + + except Exception: + print( + "Warning [vfc_ci]: Your probes could not be read for some unknown " \ + "reason" + ) + warnings.append(execution_data) + return pd.DataFrame( + columns = ["test", "variable", "values", "vfc_backend"] + ) + + if len(results) == 0: + print( + "Warning [vfc_ci]: Probes empty, it looks like you have dumped " \ + "them without calling vfc_put_probe" + ) + warnings.append(execution_data) + + + # Once the CSV has been opened and validated, return its content + results["value"] = results["value"].apply(lambda x: float.fromhex(x)) + results.rename(columns = {"value":"values"}, inplace = True) + + results["vfc_backend"] = backend + + return results + + +# Wrappers to sd.significant_digits (returns results in base 2) + +def significant_digits(x): + + # In a pandas DF, "values" actually refers to the array of columns, and + # not the column named "values" + distribution = x.values[3] + distribution = distribution.reshape(len(distribution), 1) + + # The distribution's empirical average will be used as the reference + mu = np.array([x.mu]) + + # If the null hypothesis is rejected, call sigdigits with General mode: + if x.pvalue < min_pvalue: + method = sd.Method.General + s = sd.significant_digits( + distribution, + mu, + precision=sd.Precision.Absolute, + method=method + ) + + + # Else, manually compute sMCA which is equivalent to a 66% confidence interval + else: + method = sd.Method.CNH + s = sd.significant_digits( + distribution, + mu, + precision=sd.Precision.Absolute, + method=method, + + probability=0.66, + confidence=0.66, + ) + + # s is returned as a size 1 list + return s[0] + + +def significant_digits_lower_bound(x): + # If the null hypothesis is rejected, no lower bound + if x.pvalue < min_pvalue: + return x.s2 + + # Else, the lower bound will be a 95% confidence interval + + distribution = x.values[3] + distribution = distribution.reshape(len(distribution), 1) + + mu = np.array([x.mu]) + + s = sd.significant_digits( + distribution, + mu, + precision=sd.Precision.Absolute, + method=sd.Method.CNH, + ) + + return s[0] + + +################################################################################ + + + + # Main functions + + +# Open and read the tests config file +def read_config(): + try: + with open("vfc_tests_config.json", "r") as file: + data = file.read() + + except FileNotFoundError as e: + e.strerror = "Error [vfc_ci]: This file is required to describe the tests "\ + "to run and generate a Verificarlo run file" + raise e + + return json.loads(data) + + + +# Set up metadata +def generate_metadata(is_git_commit): + + # Metadata and filename are initiated as if no commit was associated + metadata = { + "timestamp": calendar.timegm(time.gmtime()), + "is_git_commit": is_git_commit, + "hash": "", + "author": "", + "message": "" + } + + + if is_git_commit: + print("Fetching metadata from last commit...") + from git import Repo + + repo = Repo(".") + head_commit = repo.head.commit + + metadata["timestamp"] = head_commit.authored_date + + metadata["hash"] = str(head_commit)[0:7] + metadata["author"] = "%s <%s>" \ + % (str(head_commit.author), head_commit.author.email) + metadata["message"] = head_commit.message.split("\n")[0] + + return metadata + + + +# Execute tests and collect results in a Pandas dataframe (+ dataprocessing) +def run_tests(config): + + # Run the build command + print("Info [vfc_ci]: Building tests...") + os.system(config["make_command"]) + + # This is an array of Pandas dataframes for now + data = [] + + # Create tmp folder to export results + os.system("mkdir .vfcruns.tmp") + n_files = 0 + + # This will contain all executables/repetition numbers from which we could + # not get any data + warnings = [] + + + # Tests execution loop + for executable in config["executables"]: + print("Info [vfc_ci]: Running executable :", executable["executable"], "...") + + parameters = "" + if "parameters" in executable: + parameters = executable["parameters"] + + for backend in executable["vfc_backends"]: + + export_backend = "VFC_BACKENDS=\"" + backend["name"] + "\" " + command = "./" + executable["executable"] + " " + parameters + + repetitions = 1 + if "repetitions" in backend: + repetitions = backend["repetitions"] + + # Run test repetitions and save results + for i in range(repetitions): + file = ".vfcruns.tmp/%s.csv" % str(n_files) + export_output = "VFC_PROBES_OUTPUT=\"%s\" " % file + os.system(export_output + export_backend + command) + + # This will only be used if we need to append this exec to the + # warnings list + execution_data = { + "executable": executable["executable"], + "backend": backend["name"], + "repetition": i + 1 + } + + data.append(read_probes_csv( + file, + backend["name"], + warnings, + execution_data + )) + + n_files = n_files + 1 + + + # Clean CSV output files (by deleting the tmp folder) + os.system("rm -rf .vfcruns.tmp") + + + # Combine all separate executions in one dataframe + data = pd.concat(data, sort=False, ignore_index=True) + data = data.groupby(["test", "vfc_backend", "variable"])\ + .values.apply(list).reset_index() + + + # Make sure we have some data to work on + assert(len(data) != 0), "Error [vfc_ci]: No data have been generated " \ + "by your tests executions, aborting run without writing results file" + + return data, warnings + + + + # Data processing +def data_processing(data): + + data["values"] = data["values"].apply(lambda x: np.array(x).astype(float)) + + # Get empirical average, standard deviation and p-value + data["mu"] = data["values"].apply(np.average) + data["sigma"] = data["values"].apply(np.std) + data["pvalue"] = data["values"].apply(lambda x: scipy.stats.shapiro(x).pvalue) + + + # Significant digits + data["s2"] = data.apply(significant_digits, axis=1) + data["s10"] = data["s2"].apply(lambda x: sd.change_base(x, 10)) + + # Lower bound of the confidence interval using the sigdigits module + data["s2_lower_bound"] = data.apply(significant_digits_lower_bound, axis=1) + data["s10_lower_bound"] = data["s2_lower_bound"].apply(lambda x: sd.change_base(x, 10)) + + + # Compute moments of the distribution + # (including a new distribution obtained by filtering outliers) + data["values"] = data["values"].apply(np.sort) + + data["mu"] = data["values"].apply(np.average) + data["min"] = data["values"].apply(np.min) + data["quantile25"] = data["values"].apply(np.quantile, args=(0.25,)) + data["quantile50"] = data["values"].apply(np.quantile, args=(0.50,)) + data["quantile75"] = data["values"].apply(np.quantile, args=(0.75,)) + data["max"] = data["values"].apply(np.max) + + data["nsamples"] = data["values"].apply(len) + + + + # Display all executions that resulted in a warning +def show_warnings(warnings): + if len(warnings) > 0: + print( + "Warning [vfc_ci]: Some of your runs could not generate any data " \ + "(for instance because your code crashed) and resulted in " + "warnings. Here is the complete list :" + ) + + for i in range(0, len(warnings)): + print("- Warning %s:" % i) + + print(" Executable: %s" % warnings[i]["executable"]) + print(" Backend: %s" % warnings[i]["backend"]) + print(" Repetition: %s" % warnings[i]["repetition"]) + + + +################################################################################ + + + # Entry point + +def run(is_git_commit, export_raw_values, dry_run): + + # Get config, metadata and data + print("Info [vfc_ci]: Reading tests config file...") + config = read_config() + + print("Info [vfc_ci]: Generating run metadata...") + metadata = generate_metadata(is_git_commit) + + data, warnings = run_tests(config) + show_warnings(warnings) + + + # Data processing + print("Info [vfc_ci]: Processing data...") + data_processing(data) + + + # Prepare data for export (by creating a proper index and linking run timestamp) + data = data.set_index(["test", "variable", "vfc_backend"]).sort_index() + data["timestamp"] = metadata["timestamp"] + + filename = metadata["hash"] if is_git_commit else str(metadata["timestamp"]) + + + # Prepare metadata for export + metadata = pd.DataFrame.from_dict([metadata]) + metadata = metadata.set_index("timestamp") + + + # NOTE : Exporting to HDF5 requires to install "tables" on the system + + # Export raw data if needed + if export_raw_values and not dry_run: + data.to_hdf(filename + ".vfcraw.hd5", key="data") + metadata.to_hdf(filename + ".vfcraw.hd5", key="metadata") + + # Export data + del data["values"] + if not dry_run: + data.to_hdf(filename + ".vfcrun.hd5", key="data") + metadata.to_hdf(filename + ".vfcrun.hd5", key="metadata") + + + # Print termination messages + print( + "Info [vfc_ci]: The results have been successfully written to " \ + "%s.vfcrun.hd5." \ + % filename + ) + + if export_raw_values: + print( + "Info [vfc_ci]: A file containing the raw values has also been " \ + "created : %s.vfcraw.hd5." + % filename + ) + + if dry_run: + print( + "Info [vfc_ci]: The dry run flag was enabled, so no files were " \ + "actually created." + ) diff --git a/ci/vfc_ci_report/__pycache__/compare_runs.cpython-39.pyc b/ci/vfc_ci_report/__pycache__/compare_runs.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0be4ed463e7dfb486022766db9121075b31904b2 GIT binary patch literal 9285 zcmbVSOKcohcCBAmSAW_3O4PS)OOD!+DM~Yn9a&arOYtuw#)$+^&@-NLv0sravb(GG zs#>CYnwcQ-e4GpcPbNPEUXTL>fs!DLAjoRgS1T{n1>%m4> z`pRx}r3f3S&o{zWCv5xeII888eamn9jaXH#hs|!QeZz~rTVYo=e3g0IYx_-Q-S*ZZ zm3t$+>&x3=*i^aO{!aX6yVH$T>H4PM*j@{FUYB9Fqq4WX4tH^`ccVCLy>+WW%#8mS zD6HTwdLP8s>blTyRJt(U*XpJ)g@x1-8DS&Mh^%ms+9D_NNVB3Kibx$%5@n=0F(oQU z^I}@eAT5YlF^9A$=EVZik~kucA}xzWaSZ8{I4({gt%#H26w+yN8V6QYh3hyFPx`;# zZAT4aaPX5uh9uxR{vt=$v<+?7h(QOxLf^~mntg3c-?jRdFt&|oxo^crUw@!)Tk^#u zw`;|wF#6U5L~W z!k9dgdK?(K);C3FQKNqSY+AO{vMrEH>$1Z-WyfrXe17uYh0U<#U-0m@nt|WGa3gGVTlA(bd^MEY7v7fPhV*bE?Trh!HhtOh+Rxt% zr3|95?XFXgMsVTodc$o5uJk*hj4#l$gcS02mpb=pcWrmT;7P0*od?lOOE*TpKezL? zKBqfo**L1tnT~$Ote{N43uf==m}D=FPuHNU82pf!(=CuXi>}}SKsX`Npm>>!+LpPI zpH)22Q9e+?S}RMy+sbsp3=;=A^#UGvote$XOrAjEza}4=CH5A16|B- zTOaE3wYa!n>f;GQ{PkT2rE*g0I|q1Tee5WyOE|@RT%VM%+brgs5(~*%u6E~-u`>1m ziXDK0e^77_O1$$NVyvDPzb>sfAn_OaGqOIy!zYb|LN8Wh8GtEzx!28-gxcx5V| z7VdgdmDBv1*Vy*kLgj`pQQ7G;)OQX^CP}1?KT;1-6(qop-aQKW(s!X``^NV)C{XR& zkjx(%cdT35Lw%K>XU$Z`(y}tQ{rf6&*K2nDhq^qA3Ak0(kW=}qq-f~XUHb;gKf|Ba zJ3sznx*gJRZl@W>(bCms*zlUswWZN8tdfa>jg|+^aB*1~(WRbsHgb0o#l|Y`uy#{U zJA&eA`jPxitR}C}l4T5OhWC8wu7zD3%dxNbT@RWu)Khzdlq=~5R-?Tf21JRC zHlbit{!Z6x$3fG-ba|A&ygbUka9NoxZ$}xeuChCP9JTTjnUu@K+RH?)5&115D@0x) z@+uLsi1H;6RfhGa0~&CYP0P4>IvGAZRZQ|J)7hl7qa2)C(jiZ|v^hih9hHqPC2cum z(k2Vpo!Sgr9vnJuL}l*<0%jnRUnZ8GBbEvg)Tztcj8rxbT{b9oa+~rfb;$&=--_fS z28AJt3Jk6?dx+{7qlSWId;`4mlRkGK_(_pqF6|PM9+pTTg$DVJ@jL?EkUni`FTfeh-_C zaV8oTF_8XHo*pO3LiEodF%ZZDJ=URQp9S(kUJwd;8n^^zWY=t4q#|GpU_1mF6KmU) zuZ_xX(ynb~fQ`t+01pZCKqu3%pFhCUcMBLxe*lV5b;Ut$%cAU!mUUnfmWH~rSNTcFUkCfjfszWmwWcr0&?>D~R#tq!<14Ef!19x@$loKv zqV~I#dz(6Dg0}E?RE5w;)4%IC2eLOMWY}>l-c(0CE`KgBN2~6I*Rn{k5C((0MYECS=94}WmJr!Zkc7> zcvLpZ){H)9_l`_ZZZ!JI*ZMD*aqL>>23N_KX7DANxWq4y%C5F$={w6jU5SFjmATY~#Bae&y8){E4mRnqtFSZB=$mZF(_wZt~ z>BZ7bCbwH4w3V73`8-v>3sNgdQvT8A`j29820HT(&b z=jWpeNrAN?>*Oc$tV-$mCL{ivxF@}%LD^M|-r|HEBt4(Jfr-Z^L(Dnp(9~kez(7%t z3^Rd-;su&EU@OD&Db07FPRI-4QL#z!IPf2+7z!g zjny)k+u>1!Rq?*X4AAE+@HWkz6LVsI51|d@2eYGJI{N}V@kx7G;MtF2c6&JcLOMH; zuQ&!gPqXJS`cg?d ze8o@Ak6z+woz2E3~?eo=pn?BTkFAQ7f zt2^U``{M-$4(F?pch~RDY=pPNH``GMsDlC!=V~v>--cN9PQQ~{qpBe56RdT~SX3zt zQzdW_yl(;ze+Z#rMF!LZctd=QsUeV>4FSA$W-q z*@-Ym&?C6n3WZ-QJv4gx*Mqp)V9&l+UOD9HKQxquXp!H`tPo0mXwP^o!o8zTb^E3-04TAEL3+N4Ub0W*4~cw>$RB~!3_hEmKIqvijN;`3Y9eqd zUj(^bv*fp_@Es!G1yPx1(1ySBCi1;gw}Op!unsiTjv>|v8%5Pg!7FK$M^Mh|t zGd?LrRrZ*w>@oE!R6XuPC0xBlJdmGN%RI)E{2prSg)y(ImwnZX{3^|kz|m&V6tYXh zze2-5i!J#9df=|2OEp9O3FW7d|1>i;jH1UYw{03ltmA?UW9Mhgu}?bm;5RZo9ZF9a{CsBc9vGh>de@#$8BR<(8Gm-b@_ zeT?TrsDWgG&i*?7qHjP~!5+aT?ju?UOl<;F1AzgH5&otyGg*G1ZlG!ELk`ub58;5Z zJ}70WFQatoO(=cT+YuSy6JQeH5kurYg4^ZhIChrXFC0+xj;;a#nCF1610%CAoM(q*q*jEb3lP@?#*>mgSvPYhy`)~JzB9WDm)ib z^qLdpcs_>P#~ujJEN0P97jt_?I^SSb^kKgJ>QDThaBM(h&b z8n7at9;bsjjvSf;u{~Okc}6~+o70AU7&>XX54~rAL;2W9-0?uX*e(P z>AZ`~+cEm?v7rMsp4=nnyDW|$8jHB%a4hl|dkVdd(^y6~r&r+z!%a+F<*!5PWD}%! zD_>225_})ZrPQOXzFbXnOGymtoL_zUTGd|)+Kpyc_|dt>nj5ZdEhVw2+DpTkJ|^M* zI75Vhan0%#zT&~^e~B?qTDY7PirjK>IVm}>G@G20^BBlJU(db`wyRfQln4$b8l9bY zpNhkrD$w>)RNpJnps1QKeg#^6I2L&th`1F;I710zVR>>RQdR?qNp{fK%iRc~PSd+z zy~1IHUVezO3sKQ1@wv#4O59#ji1LTjoWmE2#wLAek=s;B zPKqi5VT|g_R8PSxDT%BTA)GP3iWKcY+zVRwQ4tXkXzE`%)=Y-A zaMH^8VbAdr^3cZIE3ey+4!f6!I6LHC(m|5wvQ8!>)W>^6AgsvI5Cp;y03)rZ^(p?L zQ+TUSANDv#vXAd0^aA;{8LJ;Z!VhGYtUBMavRilq_~2FA2x0Hs)a2Ie#JX;wA50;K zC0&Xc)$JWlIe4KQ!UO!#?J=V`H_3GWjLw^$o`#P}7F_H(qgNgGY$pvmbQBa9qzBAL zF@p>RC@63wjA3}jTxDd-p=N|D{vN#skuQrPUot?Aq>Vi$9r6dLvtaWEUiZ1r<86R| zz6g(uYu_L){(;CpPT=RyQI{R#=h@HYhoQ|E;f5jP>SuEEmuSt8ar4aQb93p7u<{mh z@jIW%%3q+hG{(xA&u8W0a)ShC@I!feS1d@I$l!|)zHgDoj<^#nnTwC=pk{0Zh@9-{ zPfCTsNF_sGk6fE6a1TWv$r4ipl2e43(>G}1|0nq6;T4hxmN+=PX}Sc!+P|Y()|sq^ z>GQ1265&TnKs`~be?_^!B|;%NI3lDKCysvXaXK!ZIXUi@xf<0?fh^!Js)9IK=Q-zV&Kc*lV>=E?-_k11mNV^` z{A+XX+7WpQec?ZoqZ;8~S%k9S>~2aN27hzf_BWTvb%_zKNSq^5JN3d^NG(2mav@26 z6dVq!p@;_TCy7&XPY&hCN0E6VWaJX-RiYdlIQk$YKO{msn6x$JTuqMj8tMM10)o#j P_zY9QCz|tgGui(KS}F1J literal 0 HcmV?d00001 diff --git a/ci/vfc_ci_report/__pycache__/compare_variables.cpython-39.pyc b/ci/vfc_ci_report/__pycache__/compare_variables.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..88a3c9568007af9c7b43c6a1e11057fe702578ac GIT binary patch literal 463 zcmYjN%}N6?5KcDPb&F90NxSR;d17$-l|nIS6zK9Tel zt*FeGs4Bjp5>+@BOQn?v$rHVkR*01N7a&6g=nbep2M91+W-tnHn@RdcUr@mB*)#eW z>0oreh-71JBx~*>s%WGwH!9|#2^!46b+6)y?LuQ)UlEsLC5%_%Hw2h_i8scRx0fed(IA_PTLTG&k<-J>&H!x VU79Nk z)ur{U=mEnT0ttbfLdaU&i6_o`oYG8zpS zp6C3JpZ#^8v40cu>f?d=5dZoM5XmIZ*{mJ;jDr`sD7=}+TbVx#q&EwtKkLarhF`K- zBzrQ#*q41dz&Mah4dNT}2Ix=@yH&cU13`%dy=1lFxSKzI+X`u{@BkW89Pc z5t|%d!3#E>a2usnvCyinuXZ81A~#b(%I={#{XfFLrVtzUgf;v&d&Iutw1Jmb=|tG5 zP$rRyNo?O_g{nTpytrnFw$7d>EDE;>0nH|oRiP!JmBPX5r zT;ISvaDmePVy?UZYWYxP@w;c6DHr|0~7oyf46RgO-_zh>P5ty@Tsn@a(`fN1> zBa>&Hi2doU*6eeo&Rf}P2V+r{=B(u{uP2cn<5f1C7p}>oG|5#d6KzA{=S6!s^sU)SyG=_51U!9tUgCaXQ!vutBtn2wBaIO>LkC` zw)!qsejopO3c^Ih2k6#_`Awr1s3UP;T4*&=lp-drM;80z` zs`fz`54nC5!h43+dEBCQ|TJet4iB1l8e}6>3MB~i{v7! z?dwJIq|#0n=UJ%$R(Yi%&dP5P-)XJ${}w={8`tpHfoU*wh1Zz00#IUBV( z>zrD`qNZv9tx&-Tbb#7MLKD(%cchr3oWviXlr0Ja#{_1` zQHy9mF9Gx-z2C4#+#=io!l4G<0XuhShBol0S5%@;Tr*4pJpp4yxv*iOWR{fF+R}Q- z<9ZU>AkjK`W(SnaRh?E^>33nT-=js5%y6HTshaInOM@#+*Riv|o|9dvj7m+sw)#C* z*Z3MhR)%~B=K-gK`#2dxp??bLR$jbQ^auYn%Oi$MpWpTu+AK01JiDw4w)O~$xS zevH9J>(_nF9j%J$soJ>n7g(>aM*_B#%CLTDzIpw>z)%^`x`6)MlweNIpoJYzPOUk* zEHTH0LzbP!ah=BB(m1Zu1X~)%b((NX~OUW-RWAJACsK`XTMTuW!@d4*vTYkla*N zUc0WaLAfjz&-6QF@IxZ+6QR!`_XTks4+-1cA^pGW?j5iA>LAkrWpq+ha+#}#)GhV* dKyb#<&ynYIKa7HCC;A1j+l#_zf4Cd@{{!t{cwhhk literal 0 HcmV?d00001 diff --git a/ci/vfc_ci_report/__pycache__/inspect_runs.cpython-39.pyc b/ci/vfc_ci_report/__pycache__/inspect_runs.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4c7717c0c780adbd0504d5ea6f7de8a68e003505 GIT binary patch literal 10125 zcmb_i-IE)~ao=w&fCUcs!TTU73X(;MkR_6$L(7pVStZG_>_it?RGca}%5o30cemgI z3(YQ$M;MnlA(d3hrlV9+DVL*E;qs7#`KVO+*?&PEoILMSs#2ATp7M}NC1DgXzwTY& z0Pq+~RXWtpO!xHk%ua9jboZQBsgxA_{rK$nu08V0it^6{O#kTsUd9vu0EJM58Y&&N zsjAf0Lan1Wb#CjS(KLb6L#tUpZG?8G*erHR%@X&UVY%Zp9d27;rBiKII}6Q)POVw% zEH)QAOU4BXr9P%LV8c|{a#vmb=PlQ--`A# z(0;WaCsF4cS6j5I=|2s?%Xs1;3SVieLTPG3720j3sS8~gs10EX3$>ZOWz!NxQ36~L zW#OQ)z_4FmJYO`*4BpV5DVW>t|;$1 zLlq3Q&D7|69lxQccE?XVfsIU!AdFhF1|C2gu^(=yMTz-b5n#)88Qr+6$W?UTyLMqW z>i8Eth$;;H?uFN)R=-0cy6~%!ynf+L8SO}~(+Rpe7q0I5vg381eIt@Gh@-B%O%$!* z!rpevZ3V9Mdyz~o1ljAkG~fB&&2(wU?=s+G9eD2`>Yk!Ev{5LUp*w||HaIi2Dd!&? z&>{hj|EOKW!}}!AAVCh*Rb^WvQ4ZBLMVM;@Svgc-$j+f6L!}%lGvwq@g`sK=EihEu zv9ZsK*yl~`^T2%hZ0zo*`Wr#_p8E2@aj`kyIeYQ^b6@fgfc5qX>P`Tvn_RdW8f6E$f9* z;wDhVB|CnKEqVwUyU} ztPU*5$KD(iNHUNHy+_bZwsQm|Ql*}wjh}@tUz~-XzdW#*Q$O2-I+irQC66ArC8Q7? zFG+kEe+gsaDhfrj)yePP;PXFgXQp_}Y|@Nzcp7FSQATQ_w6#NQ78wMT+6Juu9jJm( zujKFu3L(tf3R7u>ZDTWq#r@WXg8ss!A6Q|-)cie64QQBWaj31r*pS&FYRqU7uOaiA z;4J1}nx<_NE!$3BQRSGvs&Ke(;h4T!-nTg2$Mf4f*U|&{Y$ukF@GmOwHrYh3Y^GYL zpW5SZo=&P8ez3Ef%o<=a#-vJD@vwPHl%aA9M)t0{c~?#K(C?yBAJ7l`O6YaAg!j@f zVc@^xQ3lUW%b85F+mlhtk7F3p^H*S;y)b?WjciF|(&8S@j7SCpKP`B01-u=22wtxj-b~G) zEBt+&Z#Z^RofO|5>o`DfzoBPltw;jc#xds0$FS(RS5Ft_2+gOFF}K? z`?h6kHEl&(S5NbAU0qjgeQ@gkk@Ji^D_}>~l8u}JZKi{r^tR1MJhvc0KJa1o(9)O! zxSWDoM4o8^AId5um}HgV2SF`T2+Q-R>RIL=OcA1VjsVR@*&sG!teRFaipd353sc1Ns1 z#&GgRtDsSbI@R#SA0bW0SKu(-5od<@UWU%L4*4kUHSs9+Y2Dor^*Q`wIsWlE{O2Bm zeVZjmDr!uS{J9Ta|t?ZC1zjf1<r0vQ{=!PC?F z-0#6M_%69U@qO-8H_xm*SS3A8x1~|Iw`9W0w=}*YJJ`2a1zv7-R>RUGf zKU0S*SogvZ?M7MxuY3u8atyB~URQYV>V>}-coY!T|KZjrVN~HmdCY6@{MBHm8*F1m z-J~vpogj(p@h9O^#;HE$l9u92lXfASU?0anDClUtmi6|8APHOIXVIPNguIH)l?d-_cslSj!JWf7MkhSZF zJzwHD_7fcW?pTiLn)EwJ0!$*1Nk<_QP@EbxXS$HnP5O;i)VnE>Q^8WM`!{3Qj_3wv zFj=EhGYOK=PmRRiPtuC?V?S~CVTQUpext&kLvvwloJ0`!Lhq*Zz^qxD*r*(x+G9*= z@ralm4Xg{{cGF3s-sEkX3tJHq7@Z?T8DGbE$?_fIZ&S*{#n&_MsW{UXDFkW!LlhK% zmF^pGRPx$15OO%G_OWr_Ftz_N3WjQBKi&Fo+o@T$UQ?%jn*E& ztKjIX2<%qW6|2PKYr2hbwyvr6eeJ$uKBO+Js`MMIKg^i6$g>^)70@v8uxIp_XyCLt z&NCuUM;{s!zX!g~P;ue0u=U@yIaMXQiTr%Uy8g7KE13>`3 zy%742FkWtgI4wjyvLA6;$v=(kL%dtaJYO5psHE&rsz;$n^{#(IzDwhZ05i^VK$Ff! z18FJehUHO)FFuW8@|o$5zK+LM@d;>-R#TU(!Rj%h#^4!2+u$(~^xpuWkfyDUG;(&} z%(8&l5jNlqWe|kPBGyT~Z4y$}qR)UJb!g@-r13yuL98TfdsJjMwgh2A;JFjWG_J#f z&*&Wp`$upiGht`_77M$=!e&=;EbOmi%`EXFG0Q#R8dWwjnS8Gk5euGW&eFd|IF>L8 zSMF1BlM0eyCV=mt)v#G=@_?|vLB(&PNGoIG;__)o@{`}C;yB4wbIIj~tB4~nJ_BM! zryPx==O$uaekYHSU(RAihBL}ajth4CXDcgmIgpLP5<{pmDzGBjkWFz)5lPkvKA0oX zmmnJnMPxFp*FiX~ z95aeB6Clowp4nM4i5ea@gAN**6PuZu+>wlzLL3>O8TB&rPUEnMICG$BjD`2 z`jo4X?}49We{gPdHuBdSA7)1m*z1#p=!3JzEabrvGp;bj&P6=DH9rO~p&lRuA+$eJ z4>;u;s1y~CkoHCV1;a5?xrI*&IOL|tC;&$-oD0Ae(6Sjvtzx1fB1X0!nf@Z8XH__& za)(Y3UD$}-jNI_pd?FwgMr4Sb)le6;JElr1P(Y;3)D(##I?`}Aa6G?7=2v9u#hl*~ z_?a+xrdXci2a25E%0u|AOz~UE`BCIbryHVU;+K?oHuQ>Bv35s8gsO@4hwxjU;>1>(_i(Y%(BuZhkIXxz$pDc-82P$F<7)_=M7O3`gE~;n*g-jRN15$S&y85~sNtpO(43%elg|kUNKP9FYT$!66t> zbkT`~FTY7soJE1FFMFILPnXB7skE+qn`Ut`PdoEq5q>;I<9|TK?^E$BL{lOsJfj#C zUh!hT{@g{dH(R?wD0s2*TSQlcm!6YZOm-=)#%_Y^vx#GG!fTIiU>>>?g3C)+YZa!T zTqL;=4M#psGe1EE86(+YQrZEy`9T=EB1#C7l>AF|TiP)};f;%EuaGJ00eHCwx-`QT zrhmc}?VRiStwQ_c=oEXpr{PUKLf#bi4;Rw#n-;L=)?^Qt zXXag=niuVS56|%1JW3bYJ9NN~cjCm%SR5~tHKU!cxlUu*Q(oIl?LAKh-WH{8ru_H|julo`qBFBwqu+hAJ1S(o*CFV6iC^BiW^gB2os3Oyg9v z#_l`CckEUU7O(PLb+4YeP4yS#o9G-Y`~oK|>ti3L@t7oskmI~8BA^6>6yV=TX~B(;^d zsNgrcO)ZW}UZK`pEJALhB!5f(E()BB%czxM&60U9yLHcIofPB9dpR6iUb~kj1l0Rb0E++PZGQVi~IOv12SDBT=PZ zz*XcSjF0UKK~?HQfCK!;hK+ECZ5)r7VDs;B znP*lyb2STVsKBkrY$JU{NXO?<5v(JnxwTDs2s}KOCMy0e(Wb3UIb}u>`h!QsjUZ|5 zx_g0t<4H_0cI0Qx^$|_W1iSH{g$6xru4zYUNv;B;3WJUOKMXYm*sh)!0leuu>ceWG zeL!?(K<3-xl00aHZe$86d?Wvo4Ko=*=TKt>66Xom#&GL9L_@?KjW4k!_gT!^8CDGI|TG5a* zqb!+`U-y4x7XTqmVw*L!Y7ADU&)M~!p3I~? zPK$hkir1;2V33VOrroUAtiPnX@@OmW68N`N9C;($S|i*73W7MtW1IG(?bsFjDf_^# z<98C6lKqZ-!!9soV*_{9Ua~9cqfd+5Nb#^UEJ*?Ie;1V4j`81MA~n1H4zjN=5x%+= zUH5m-lk=FsW6s=5`d*vmtjEVcst8G6Z0ckfa;n2vvWa9#Ph3h_23cm+*if-eWO4tH jI_Q0ld(qUslI`tFv{e4j6-RaGzcY}*a@2DqsKWmN<#CWd literal 0 HcmV?d00001 diff --git a/ci/vfc_ci_report/__pycache__/plot.cpython-39.pyc b/ci/vfc_ci_report/__pycache__/plot.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9e4b5971eaf6cb3b45368078cade265fce44ad32 GIT binary patch literal 2455 zcmaKtTW{P%6vsWj+j~uuCcSWn6c%VVEkxW(Thyu`BqBsBfmR6P^>}x^vDeG7H{C?e z8&61)_z3MA`35}lA$S^Ika)^hCriF>0%^nrIy@(K~u#bPQ?JVl%Ni7Pc$g-Usa?FeU5T1=R|s6T)6&07&i{szAGbK8DYb;P=*FK3w2c~)LG3f*<;-1&VjjK!3dL=zCtY+cX{Q&*sm5A z{DeMhJ{mn$+)u4A<+{HV)uBCf9=aSYk5oE3jIMZ9S=X?xW53Sp=apf@Yo6*Y@y3Gh zOuo#U7ishgMSEwtf``1cC5)Xo8im3PA_(H}P!o2J2b@RaOz260TW*P9KEeb<@d+3!@vO^UxIPbVI1S_)M>2eN~uQI)NY*F0JSV zX`I4Q6>p|x2%Q!YY%16;tf5KhS+s|D4i~Wl&yYy76^(clgc;ynDn}X#-EV6Zs4d)H z3W=ZRA!NewIrr#YGSJZn1_fAI*iXVy4m3*W^3^Nc&wa1lo1KMm7~~-rPCD{}fj{cw z5ls6KanT_4PBt%$U1>LV{b_^?9xpDMV|gBNltj5Vi24K3ShV1231Ykr)rCHe(_9`y$CX>|MPbZ=5>qVCFU>pNL?#-;%**|; zH@`2joP~fHblPap3uU5m>Iw&xX+p!%o!ZQ?cyRCvu6d}y`&d2xsy#@Pu`3W6!JE` z=auoUsP>{b_IR3;fzmpA7e#Sw)>Lg~s;-VL=IT1rna!$9|I4&>rn2harlqqL^5SEw zb%s9`uH#st1QyLv7UBa-1 zxFdDec1vDVF#k>D(MeQoA*y~V3MnM1GMk~n?lJoAVpIoC0Wg``)=Zo z@+c19*@PtuJVSxxjz){_Yny#^VEO%K%YqBokZcmhgNWw?OqFyj%x;?F$4KTVoWqdC z2{t!c<_u>yY49zQk4Ok?vuNaLGC6t<`z>REe9z9Kgdw2`6d?%+&r|CKl#+n(A{kyH zIffp*j9sDcbbZWd%qCbT_a%~7NnRsqk-Sdw2FaTwmr1UWv`Gk4ut9ParB#P()RtH_ z5oWz;Dp>??k@;E4p{lYao0fm=%e literal 0 HcmV?d00001 diff --git a/ci/vfc_ci_report/compare_runs.py b/ci/vfc_ci_report/compare_runs.py new file mode 100644 index 0000000..27ff0de --- /dev/null +++ b/ci/vfc_ci_report/compare_runs.py @@ -0,0 +1,556 @@ +# Manage the view comparing a variable over different runs + +import time + +import pandas as pd + +from math import pi + +from bokeh.plotting import figure, curdoc +from bokeh.embed import components +from bokeh.models import Select, ColumnDataSource, Panel, Tabs, HoverTool, \ +TextInput, CheckboxGroup, TapTool, CustomJS + +import helper +import plot + + +################################################################################ + + + +class CompareRuns: + + # Helper functions related to CompareRuns + + # From an array of timestamps, returns the array of runs names (for the x + # axis ticks), as well as the metadata (in a dict of arrays) associated to + # this array (for the tooltips) + def gen_x_series(self, timestamps): + + # Initialize the objects to return + x_series= [] + x_metadata = dict( + date = [], + is_git_commit = [], + hash = [], + author = [], + message = [] + ) + + # n == 0 means we want all runs, we also make sure not to go out of + # bound if asked for more runs than we have + n = self.current_n_runs + if n == 0 or n > len(timestamps): + n = len(timestamps) + + + for i in range(0, n): + # Get metadata associated to this run + row_metadata = helper.get_metadata(self.metadata, timestamps[-i-1]) + date = time.ctime(timestamps[-i-1]) + + # Fill the x series + str = row_metadata["name"] + x_series.insert(0, helper.get_metadata(self.metadata, timestamps[-i-1])["name"]) + + # Fill the metadata lists + x_metadata["date"].insert(0, date) + x_metadata["is_git_commit"].insert(0, row_metadata["is_git_commit"]) + x_metadata["hash"].insert(0, row_metadata["hash"]) + x_metadata["author"].insert(0, row_metadata["author"]) + x_metadata["message"].insert(0, row_metadata["message"]) + + + return x_series, x_metadata + + + + # Plots update function + + def update_plots(self): + + # Select all data matching current test/var/backend + + runs = self.data.loc[ + [self.widgets["select_test"].value], + self.widgets["select_var"].value, self.widgets["select_backend"].value + ] + + timestamps = runs["timestamp"] + x_series, x_metadata = self.gen_x_series(timestamps.sort_values()) + + + # Update source + + main_dict = runs.to_dict("series") + main_dict["x"] = x_series + + # Add metadata (for tooltip) + main_dict.update(x_metadata) + + # Select the last n runs only + n = self.current_n_runs + main_dict = {key:value[-n:] for key, value in main_dict.items()} + + + # Generate ColumnDataSources for the 3 dotplots + for stat in ["sigma", "s10", "s2"]: + dict = { + "%s_x" % stat: main_dict["x"], + + "is_git_commit": main_dict["is_git_commit"], + "date": main_dict["date"], + "hash": main_dict["hash"], + "author": main_dict["author"], + "message": main_dict["message"], + + stat: main_dict[stat], + + "nsamples": main_dict["nsamples"], + } + + if stat == "s10" or stat == "s2": + dict["%s_lower_bound" % stat] = main_dict["%s_lower_bound" % stat] + + # Filter outliers if the box is checked + if len(self.widgets["outliers_filtering_compare"].active) > 0: + outliers = helper.detect_outliers(dict[stat]) + dict[stat] = helper.remove_outliers(dict[stat], outliers) + dict["%s_x" % stat] = helper.remove_outliers(dict["%s_x" % stat], outliers) + + # Assign ColumnDataSource + self.sources["%s_source" % stat].data = dict + + + # Generate ColumnDataSource for the boxplot + dict = { + "is_git_commit": main_dict["is_git_commit"], + "date": main_dict["date"], + "hash": main_dict["hash"], + "author": main_dict["author"], + "message": main_dict["message"], + + "x": main_dict["x"], + "min" : main_dict["min"], + "quantile25" : main_dict["quantile25"], + "quantile50" : main_dict["quantile50"], + "quantile75" : main_dict["quantile75"], + "max" : main_dict["max"], + "mu" : main_dict["mu"], + "pvalue" : main_dict["pvalue"], + + "nsamples": main_dict["nsamples"] + } + + + + self.sources["boxplot_source"].data = dict + + + # Update x_ranges + helper.reset_x_range(self.plots["boxplot"], self.sources["boxplot_source"].data["x"]) + helper.reset_x_range(self.plots["sigma_plot"], self.sources["sigma_source"].data["sigma_x"]) + helper.reset_x_range(self.plots["s10_plot"], self.sources["s10_source"].data["s10_x"]) + helper.reset_x_range(self.plots["s2_plot"], self.sources["s2_source"].data["s2_x"]) + + + + + # Widgets' callback functions + + def update_test(self, attrname, old, new): + + # If the value is updated by the CustomJS, self.widgets["select_var"].value + # won't be updated, so we have to look for that case and assign it manually + + # "new" should be a list when updated by CustomJS + if type(new) == list: + # If filtering removed all options, we might have an empty list + # (in this case, we just skip the callback and do nothing) + if len(new) > 0: + new = new[0] + else: + return + + if new != self.widgets["select_test"].value: + # The callback will be triggered again with the updated value + self.widgets["select_test"].value = new + return + + # New list of available vars + self.vars = self.data.loc[new]\ + .index.get_level_values("variable").drop_duplicates().tolist() + self.widgets["select_var"].options = self.vars + + + # Reset var selection if old one is not available in new vars + if self.widgets["select_var"].value not in self.vars: + self.widgets["select_var"].value = self.vars[0] + # The update_var callback will be triggered by the assignment + + else: + # Trigger the callback manually (since the plots need to be updated + # anyway) + self.update_var("", "", self.widgets["select_var"].value) + + + def update_var(self, attrname, old, new): + + # If the value is updated by the CustomJS, self.widgets["select_var"].value + # won't be updated, so we have to look for that case and assign it manually + + # new should be a list when updated by CustomJS + if type(new) == list: + new = new[0] + + if new != self.widgets["select_var"].value: + # The callback will be triggered again with the updated value + self.widgets["select_var"].value = new + return + + + # New list of available backends + self.backends = self.data.loc[self.widgets["select_test"].value, self.widgets["select_var"].value]\ + .index.get_level_values("vfc_backend").drop_duplicates().tolist() + self.widgets["select_backend"].options = self.backends + + # Reset backend selection if old one is not available in new backends + if self.widgets["select_backend"].value not in self.backends: + self.widgets["select_backend"].value = self.backends[0] + # The update_backend callback will be triggered by the assignment + + else: + # Trigger the callback manually (since the plots need to be updated + # anyway) + self.update_backend("", "", self.widgets["select_backend"].value) + + + def update_backend(self, attrname, old, new): + + # Simply update plots, since no other data is affected + self.update_plots() + + + def update_n_runs(self, attrname, old, new): + # Simply update runs selection (value and string display) + self.select_n_runs.value = new + self.current_n_runs = self.n_runs_dict[self.select_n_runs.value] + + self.update_plots() + + + def update_outliers_filtering(self, attrname, old, new): + self.update_plots() + + + + # Bokeh setup functions + + def setup_plots(self): + + tools = "pan, wheel_zoom, xwheel_zoom, ywheel_zoom, reset, save" + + # Custom JS callback that will be used when tapping on a run + # Only switches the view, a server callback is required to update plots + # (defined inside template to avoid bloating server w/ too much JS code) + js_tap_callback = "goToInspectRuns();" + + + # Box plot + self.plots["boxplot"] = figure( + name="boxplot", title="Variable distribution over runs", + plot_width=900, plot_height=400, x_range=[""], + tools=tools, sizing_mode="scale_width" + ) + + box_tooltips = [ + ("Git commit", "@is_git_commit"), + ("Date", "@date"), + ("Hash", "@hash"), + ("Author", "@author"), + ("Message", "@message"), + ("Min", "@min{%0.18e}"), + ("Max", "@max{%0.18e}"), + ("1st quartile", "@quantile25{%0.18e}"), + ("Median", "@quantile50{%0.18e}"), + ("3rd quartile", "@quantile75{%0.18e}"), + ("μ", "@mu{%0.18e}"), + ("p-value", "@pvalue"), + ("Number of samples", "@nsamples") + ] + box_tooltips_formatters = { + "@min" : "printf", + "@max" : "printf", + "@quantile25" : "printf", + "@quantile50" : "printf", + "@quantile75" : "printf", + "@mu" : "printf" + } + + plot.fill_boxplot( + self.plots["boxplot"], self.sources["boxplot_source"], + tooltips = box_tooltips, + tooltips_formatters = box_tooltips_formatters, + js_tap_callback = js_tap_callback, + server_tap_callback = self.inspect_run_callback_boxplot, + ) + self.doc.add_root(self.plots["boxplot"]) + + + # Sigma plot (bar plot) + self.plots["sigma_plot"] = figure( + name="sigma_plot", title="Standard deviation σ over runs", + plot_width=900, plot_height=400, x_range=[""], + tools=tools, sizing_mode="scale_width" + ) + + sigma_tooltips = [ + ("Git commit", "@is_git_commit"), + ("Date", "@date"), + ("Hash", "@hash"), + ("Author", "@author"), + ("Message", "@message"), + ("σ", "@sigma"), + ("Number of samples", "@nsamples") + ] + + plot.fill_dotplot( + self.plots["sigma_plot"], self.sources["sigma_source"], "sigma", + tooltips = sigma_tooltips, + js_tap_callback = js_tap_callback, + server_tap_callback = self.inspect_run_callback_sigma, + lines = True + ) + self.doc.add_root(self.plots["sigma_plot"]) + + + # s plot (bar plot with 2 tabs) + self.plots["s10_plot"] = figure( + name="s10_plot", title="Significant digits s over runs", + plot_width=900, plot_height=400, x_range=[""], + tools=tools, sizing_mode="scale_width" + ) + + s10_tooltips = [ + ("Git commit", "@is_git_commit"), + ("Date", "@date"), + ("Hash", "@hash"), + ("Author", "@author"), + ("Message", "@message"), + ("s", "@s10"), + ("s lower bound", "@s10_lower_bound"), + ("Number of samples", "@nsamples") + ] + + plot.fill_dotplot( + self.plots["s10_plot"], self.sources["s10_source"], "s10", + tooltips = s10_tooltips, + js_tap_callback = js_tap_callback, + server_tap_callback = self.inspect_run_callback_s10, + lines = True, + lower_bound=True + ) + s10_tab = Panel(child=self.plots["s10_plot"], title="Base 10") + + + self.plots["s2_plot"] = figure( + name="s2_plot", title="Significant digits s over runs", + plot_width=900, plot_height=400, x_range=[""], + tools=tools, sizing_mode="scale_width" + ) + + s2_tooltips = [ + ("Git commit", "@is_git_commit"), + ("Date", "@date"), + ("Hash", "@hash"), + ("Author", "@author"), + ("Message", "@message"), + ("s", "@s2"), + ("s lower bound", "@s2_lower_bound"), + ("Number of samples", "@nsamples") + ] + + plot.fill_dotplot( + self.plots["s2_plot"], self.sources["s2_source"], "s2", + tooltips = s2_tooltips, + js_tap_callback = js_tap_callback, + server_tap_callback = self.inspect_run_callback_s2, + lines = True, + lower_bound=True + ) + s2_tab = Panel(child=self.plots["s2_plot"], title="Base 2") + + s_tabs = Tabs( + name = "s_tabs", + tabs=[s10_tab, s2_tab], + tabs_location = "below" + ) + + self.doc.add_root(s_tabs) + + + def setup_widgets(self): + + # Initial selections + + # Test/var/backend combination (we select all first elements at init) + self.tests = self.data\ + .index.get_level_values("test").drop_duplicates().tolist() + + self.vars = self.data.loc[self.tests[0]]\ + .index.get_level_values("variable").drop_duplicates().tolist() + + self.backends = self.data.loc[self.tests[0], self.vars[0]]\ + .index.get_level_values("vfc_backend").drop_duplicates().tolist() + + + # Custom JS callback that will be used client side to filter selections + filter_callback_js = """ + selector.options = options.filter(e => e.includes(cb_obj.value)); + """ + + + # Test selector widget + + # Number of runs to display + # The dict structure allows us to get int value from the display string + # in O(1) + self.n_runs_dict = { + "Last 3 runs": 3, + "Last 5 runs": 5, + "Last 10 runs": 10, + "All runs": 0 + } + + # Contains all options strings + n_runs_display = list(self.n_runs_dict.keys()) + + # Will be used when updating plots (contains actual number to diplay) + self.current_n_runs = self.n_runs_dict[n_runs_display[1]] + + # Selector widget + self.widgets["select_test"] = Select( + name="select_test", title="Test :", + value=self.tests[0], options=self.tests + ) + self.doc.add_root(self.widgets["select_test"]) + self.widgets["select_test"].on_change("value", self.update_test) + self.widgets["select_test"].on_change("options", self.update_test) + + # Filter widget + self.widgets["test_filter"] = TextInput( + name="test_filter", title="Tests filter:" + ) + self.widgets["test_filter"].js_on_change("value", CustomJS( + args=dict(options=self.tests, selector=self.widgets["select_test"]), + code=filter_callback_js + )) + self.doc.add_root(self.widgets["test_filter"]) + + + # Number of runs to display + + self.widgets["select_n_runs"] = Select( + name="select_n_runs", title="Display :", + value=n_runs_display[1], options=n_runs_display + ) + self.doc.add_root(self.widgets["select_n_runs"]) + self.widgets["select_n_runs"].on_change("value", self.update_n_runs) + + + # Variable selector widget + + self.widgets["select_var"] = Select( + name="select_var", title="Variable :", + value=self.vars[0], options=self.vars + ) + self.doc.add_root(self.widgets["select_var"]) + self.widgets["select_var"].on_change("value", self.update_var) + self.widgets["select_var"].on_change("options", self.update_var) + + + # Backend selector widget + + self.widgets["select_backend"] = Select( + name="select_backend", title="Verificarlo backend :", + value=self.backends[0], options=self.backends + ) + self.doc.add_root(self.widgets["select_backend"]) + self.widgets["select_backend"].on_change("value", self.update_backend) + + + # Outliers filtering checkbox + + self.widgets["outliers_filtering_compare"] = CheckboxGroup( + name="outliers_filtering_compare", + labels=["Filter outliers"], active =[] + ) + self.doc.add_root(self.widgets["outliers_filtering_compare"]) + self.widgets["outliers_filtering_compare"]\ + .on_change("active", self.update_outliers_filtering) + + + + # Communication methods + # (to send/receive messages to/from master) + + # Callback to change view of Inspect runs when data is selected + def inspect_run_callback(self, new, source_name, x_name): + + # In case we just unselected everything, then do nothing + if new == []: + return + + index = new[-1] + run_name = self.sources[source_name].data[x_name][index] + + self.master.go_to_inspect(run_name) + + + # Wrappers for each plot (since new is the index of the clicked element, + # it is dependent of the plot because we could have filtered some outliers) + # There doesn't seem to be an easy way to add custom parameters to a + # Bokeh callback, so using wrappers seems to be the best solution for now + + def inspect_run_callback_boxplot(self, attr, old, new): + self.inspect_run_callback(new, "boxplot_source", "x") + + def inspect_run_callback_sigma(self, attr, old, new): + self.inspect_run_callback(new, "sigma_source", "sigma_x") + + def inspect_run_callback_s2(self, attr, old, new): + self.inspect_run_callback(new, "s2_source", "s2_x") + + def inspect_run_callback_s10(self, attr, old, new): + self.inspect_run_callback(new, "s10_source", "s10_x") + + + # Constructor + + def __init__(self, master, doc, data, metadata): + + self.master = master + + self.doc = doc + self.data = data + self.metadata = metadata + + + self.sources = { + "boxplot_source": ColumnDataSource(data={}), + "sigma_source": ColumnDataSource(data={}), + "s10_source" :ColumnDataSource(data={}), + "s2_source": ColumnDataSource(data={}) + } + + self.plots = {} + self.widgets = {} + + # Setup Bokeh objects + self.setup_plots() + self.setup_widgets() + + # At this point, everything should have been initialized, so we can + # show the plots for the first time + self.update_plots() diff --git a/ci/vfc_ci_report/helper.py b/ci/vfc_ci_report/helper.py new file mode 100644 index 0000000..57c84d7 --- /dev/null +++ b/ci/vfc_ci_report/helper.py @@ -0,0 +1,166 @@ +# General helper functions for both compare_runs and compare_variables + +import calendar +import time +from itertools import compress + +import numpy as np + +# Magic numbers +max_ticks = 15 +max_zscore = 3 + +################################################################################ + + +# From a timestamp, return the associated metadata as a Pandas serie +def get_metadata(metadata, timestamp): + return metadata.loc[timestamp] + + +# Convert a metadata Pandas series to a JS readable dict +def metadata_to_dict(metadata): + dict = metadata.to_dict() + + # JS doesn't accept True for booleans, and Python doesn't accept true + # (because of the caps) => using an integer is a portable solution + dict["is_git_commit"] = 1 if dict["is_git_commit"] else 0 + + dict["date"] = time.ctime(metadata.name) + + return dict + + +# Return a string that indicates the elapsed time since the run, used as the +# x-axis tick in "Compare runs" or when selecting run in "Inspect run" +def get_run_name(timestamp, hash): + + gmt = time.gmtime() + now = calendar.timegm(gmt) + diff = now - timestamp + + + # Special case : < 1 minute (return string directly) + if diff < 60: + str = "Less than a minute ago" + + if hash != "": + str = str + " (%s)" % hash + + if str == get_run_name.previous: + get_run_name.counter = get_run_name.counter + 1 + str = "%s (%s)" % (str, get_run_name.counter) + else: + get_run_name.counter = 0 + get_run_name.previous = str + + return str + + # < 1 hour + if diff < 3600: + n = int(diff / 60) + str = "%s minute%s ago" + # < 1 day + elif diff < 86400: + n = int(diff / 3600) + str = "%s hour%s ago" + # < 1 week + elif diff < 604800: + n = int(diff / 86400) + str = "%s day%s ago" + # < 1 month + elif diff < 2592000: + n = int(diff / 604800) + str = "%s week%s ago" + # > 1 month + else: + n = diff / 2592000 + str = "%s month%s ago" + + plural = "" + if n != 1: + plural = "s" + + str = str % (n, plural) + + + # We might want to add the git hash + if hash != "": + str = str + " (%s)" % hash + + + # Finally, check for duplicate with previously generated string + if str == get_run_name.previous: + # Increment the duplicate counter and add it to str + get_run_name.counter = get_run_name.counter + 1 + str = "%s (%s)" % (str, get_run_name.counter) + + else: + # No duplicate, reset both previously generated str and duplicate counter + get_run_name.counter = 0 + get_run_name.previous = str + + return str + +# These external variables will store data about the last generated string to +# avoid duplicates (assuming the runs are sorted by time) +get_run_name.counter = 0 +get_run_name.previous = "" + + +def reset_run_strings(): + get_run_name.counter = 0 + get_run_name.previous = "" + + +# Update all the x-ranges from a dict of plots +def reset_x_range(plot, x_range): + plot.x_range.factors = x_range + + if len(x_range) < max_ticks: + plot.xaxis.major_tick_line_color = "#000000" + plot.xaxis.minor_tick_line_color = "#000000" + + plot.xaxis.major_label_text_font_size = "8pt" + + else: + plot.xaxis.major_tick_line_color = None + plot.xaxis.minor_tick_line_color = None + + plot.xaxis.major_label_text_font_size = "0pt" + + +# Return an array of booleans that indicate which elements are outliers +# (True means element is not an outlier and must be kept) +def detect_outliers(array, max_zscore=max_zscore): + if len(array) <= 2: + return [True] * len(array) + + median = np.median(array) + std = np.std(array) + if std == 0: + return array + distance = abs(array - median) + # Array of booleans with elements to be filtered + outliers_array = distance < max_zscore * std + + return outliers_array + + +def remove_outliers(array, outliers): + return list(compress(array, outliers)) + + +def remove_boxplot_outliers(dict, outliers, prefix): + outliers = detect_outliers(dict["%s_max" % prefix]) + + dict["%s_x" % prefix] = remove_outliers(dict["%s_x" % prefix], outliers) + + dict["%s_min" % prefix] = remove_outliers(dict["%s_min" % prefix], outliers) + dict["%s_quantile25" % prefix] = remove_outliers(dict["%s_quantile25" % prefix], outliers) + dict["%s_quantile50" % prefix] = remove_outliers(dict["%s_quantile50" % prefix], outliers) + dict["%s_quantile75" % prefix] = remove_outliers(dict["%s_quantile75" % prefix], outliers) + dict["%s_max" % prefix] = remove_outliers(dict["%s_max" % prefix], outliers) + dict["%s_mu" % prefix] = remove_outliers(dict["%s_mu" % prefix], outliers) + + dict["nsamples"] = remove_outliers(dict["nsamples"], outliers) diff --git a/ci/vfc_ci_report/inspect_runs.py b/ci/vfc_ci_report/inspect_runs.py new file mode 100644 index 0000000..57a1caa --- /dev/null +++ b/ci/vfc_ci_report/inspect_runs.py @@ -0,0 +1,588 @@ +# Manage the view comparing the variables of a run + +from math import pi +from functools import partial + +import pandas as pd +import numpy as np + +from bokeh.plotting import figure, curdoc +from bokeh.embed import components +from bokeh.models import Select, ColumnDataSource, Panel, Tabs, HoverTool,\ +RadioButtonGroup, CheckboxGroup, CustomJS + +import helper +import plot + + +################################################################################ + + + +class InspectRuns: + + # Helper functions related to InspectRun + + # Returns a dictionary mapping user-readable strings to all run timestamps + def gen_runs_selection(self): + + runs_dict = {} + + # Iterate over timestamp rows (runs) and fill dict + for row in self.metadata.iloc: + # The syntax used by pandas makes this part a bit tricky : + # row.name is the index of metadata (so it refers to the + # timestamp), whereas row["name"] is the column called "name" + # (which is the display string used for the run) + + # runs_dict[run's name] = run's timestamp + runs_dict[row["name"]] = row.name + + return runs_dict + + + def gen_boxplot_tooltips(self, prefix): + return [ + ("Name", "@%s_x" % prefix), + ("Min", "@" + prefix + "_min{%0.18e}"), + ("Max", "@" + prefix + "_max{%0.18e}"), + ("1st quartile", "@" + prefix + "_quantile25{%0.18e}"), + ("Median", "@" + prefix + "_quantile50{%0.18e}"), + ("3rd quartile", "@" + prefix + "_quantile75{%0.18e}"), + ("μ", "@" + prefix + "_mu{%0.18e}"), + ("Number of samples (tests)", "@nsamples") + ] + + def gen_boxplot_tooltips_formatters(self, prefix): + return { + "@%s_min" % prefix : "printf", + "@%s_max" % prefix : "printf", + "@%s_quantile25" % prefix : "printf", + "@%s_quantile50" % prefix : "printf", + "@%s_quantile75" % prefix : "printf", + "@%s_mu" % prefix : "printf" + } + + + # Data processing helper + # (computes new distributions for sigma, s2, s10) + def data_processing(self, dataframe): + + # Compute aggragated mu + dataframe["mu"] = np.vectorize(np.average)(dataframe["mu"], weights=dataframe["nsamples"]) + + # nsamples is the number of aggregated elements (as well as the number + # of samples for our new sigma and s distributions) + dataframe["nsamples"] = dataframe["nsamples"].apply(lambda x: len(x)) + + + dataframe["mu_x"] = dataframe.index + # Make sure that strings don't excede a certain length + dataframe["mu_x"] = dataframe["mu_x"].apply( + lambda x: x[:17] + "[...]" + x[-17:] if len(x) > 39 else x + ) + + + # Get quantiles and mu for sigma, s10, s2 + for prefix in ["sigma", "s10", "s2"]: + + dataframe["%s_x" % prefix] = dataframe["mu_x"] + + dataframe[prefix] = dataframe[prefix].apply(np.sort) + + dataframe["%s_min" % prefix] = dataframe[prefix].apply(np.min) + dataframe["%s_quantile25" % prefix] = dataframe[prefix].apply(np.quantile, args=(0.25,)) + dataframe["%s_quantile50" % prefix] = dataframe[prefix].apply(np.quantile, args=(0.50,)) + dataframe["%s_quantile75" % prefix] = dataframe[prefix].apply(np.quantile, args=(0.75,)) + dataframe["%s_max" % prefix] = dataframe[prefix].apply(np.max) + dataframe["%s_mu" % prefix] = dataframe[prefix].apply(np.average) + del dataframe[prefix] + + + return dataframe + + + + # Plots update function + + def update_plots(self): + + groupby_display = self.widgets["groupby_radio"].labels[ + self.widgets["groupby_radio"].active + ] + groupby = self.factors_dict[groupby_display] + + filterby_display = self.widgets["filterby_radio"].labels[ + self.widgets["filterby_radio"].active + ] + filterby = self.factors_dict[filterby_display] + + + # Groupby and aggregate lines belonging to the same group in lists + + groups = self.run_data[ + self.run_data.index.isin( + [self.widgets["select_filter"].value], + level=filterby + ) + ].groupby(groupby) + + groups = groups.agg({ + "sigma": lambda x: x.tolist(), + "s10": lambda x: x.tolist(), + "s2": lambda x: x.tolist(), + "mu": lambda x: x.tolist(), + + # Used for mu weighted average first, then will be replaced + "nsamples": lambda x: x.tolist() + }) + + + # Compute the new distributions, ... + groups = self.data_processing(groups).to_dict("list") + + + # Update source + + # Assign each ColumnDataSource, starting with the boxplots + for prefix in ["sigma", "s10", "s2"]: + + dict = { + "%s_x" % prefix: groups["%s_x" % prefix], + "%s_min" % prefix: groups["%s_min" % prefix], + "%s_quantile25" % prefix: groups["%s_quantile25" % prefix], + "%s_quantile50" % prefix: groups["%s_quantile50" % prefix], + "%s_quantile75" % prefix: groups["%s_quantile75" % prefix], + "%s_max" % prefix: groups["%s_max" % prefix], + "%s_mu" % prefix: groups["%s_mu" % prefix], + + "nsamples": groups["nsamples"] + } + + # Filter outliers if the box is checked + if len(self.widgets["outliers_filtering_inspect"].active) > 0: + + # Boxplots will be filtered by max then min + top_outliers = helper.detect_outliers(dict["%s_max" % prefix]) + helper.remove_boxplot_outliers(dict, top_outliers, prefix) + + bottom_outliers = helper.detect_outliers(dict["%s_min" % prefix]) + helper.remove_boxplot_outliers(dict, bottom_outliers, prefix) + + self.sources["%s_source" % prefix].data = dict + + # Finish with the mu plot + dict = { + "mu_x": groups["mu_x"], + "mu": groups["mu"], + + "nsamples": groups["nsamples"] + } + + self.sources["mu_source"].data = dict + + # Filter outliers if the box is checked + if len(self.widgets["outliers_filtering_inspect"].active) > 0: + mu_outliers = helper.detect_outliers(groups["mu"]) + groups["mu"] = helper.remove_outliers(groups["mu"], mu_outliers) + groups["mu_x"] = helper.remove_outliers(groups["mu_x"], mu_outliers) + + + # Update plots axis/titles + + # Get display string of the last (unselected) factor + factors_dict = self.factors_dict.copy() + del factors_dict[groupby_display] + del factors_dict[filterby_display] + over_all = list(factors_dict.keys())[0] + + # Update all display strings for plot title (remove caps, plural) + groupby_display = groupby_display.lower() + filterby_display = filterby_display.lower()[:-1] + over_all = over_all.lower() + + self.plots["mu_inspect"].title.text = \ + "Empirical average μ of %s (groupped by %s, for all %s)" \ + % (filterby_display, groupby_display, over_all) + + self.plots["sigma_inspect"].title.text = \ + "Standard deviation σ of %s (groupped by %s, for all %s)" \ + % (filterby_display, groupby_display, over_all) + + self.plots["s10_inspect"].title.text = \ + "Significant digits s of %s (groupped by %s, for all %s)" \ + % (filterby_display, groupby_display, over_all) + + self.plots["s2_inspect"].title.text = \ + "Significant digits s of %s (groupped by %s, for all %s)" \ + % (filterby_display, groupby_display, over_all) + + + # Update x_ranges + + helper.reset_x_range(self.plots["mu_inspect"], groups["mu_x"]) + helper.reset_x_range(self.plots["sigma_inspect"], groups["sigma_x"]) + helper.reset_x_range(self.plots["s10_inspect"], groups["s10_x"]) + helper.reset_x_range(self.plots["s2_inspect"], groups["s2_x"]) + + + + # Widets' callback functions + + # Run selector callback + def update_run(self, attrname, old, new): + + filterby = self.widgets["filterby_radio"].labels[ + self.widgets["filterby_radio"].active + ] + filterby = self.factors_dict[filterby] + + # Update run selection (by using dict mapping) + self.current_run = self.runs_dict[new] + + # Update run data + self.run_data = self.data[self.data["timestamp"] == self.current_run] + + # Save old selected option + old_value = self.widgets["select_filter"].value + + # Update filter options + options = self.run_data.index\ + .get_level_values(filterby).drop_duplicates().tolist() + self.widgets["select_filter"].options = options + + if old_value not in self.widgets["select_filter"].options: + self.widgets["select_filter"].value = options[0] + # The update_var callback will be triggered by the assignment + + else: + # Trigger the callback manually (since the plots need to be updated + # anyway) + self.update_filter("", "", old_value) + + + # "Group by" radio + def update_groupby(self, attrname, old, new): + + # Update "Filter by" radio list + filterby_list = list(self.factors_dict.keys()) + del filterby_list[self.widgets["groupby_radio"].active] + self.widgets["filterby_radio"].labels = filterby_list + + + filterby = self.widgets["filterby_radio"].labels[ + self.widgets["filterby_radio"].active + ] + filterby = self.factors_dict[filterby] + + # Save old selected option + old_value = self.widgets["select_filter"].value + + # Update filter options + options = self.run_data.index\ + .get_level_values(filterby).drop_duplicates().tolist() + self.widgets["select_filter"].options = options + + if old_value not in self.widgets["select_filter"].options: + self.widgets["select_filter"].value = options[0] + # The update_var callback will be triggered by the assignment + + else: + # Trigger the callback manually (since the plots need to be updated + # anyway) + self.update_filter("", "", old_value) + + + # "Filter by" radio + def update_filterby(self, attrname, old, new): + + filterby = self.widgets["filterby_radio"].labels[ + self.widgets["filterby_radio"].active + ] + filterby = self.factors_dict[filterby] + + # Save old selected option + old_value = self.widgets["select_filter"].value + + # Update filter selector options + options = self.run_data.index\ + .get_level_values(filterby).drop_duplicates().tolist() + self.widgets["select_filter"].options = options + + if old_value not in self.widgets["select_filter"].options: + self.widgets["select_filter"].value = options[0] + # The update_var callback will be triggered by the assignment + + else: + # Trigger the callback manually (since the plots need to be updated + # anyway) + self.update_filter("", "", old_value) + + + # Filter selector callback + def update_filter(self, attrname, old, new): + self.update_plots() + + + # Filter outliers checkbox callback + def update_outliers_filtering(self, attrname, old, new): + # The status (checked/unchecked) of the checkbox is also verified inside + # self.update_plots(), so calling this function is enough + self.update_plots() + + + + # Bokeh setup functions + # (for both variable and backend selection at once) + + def setup_plots(self): + + tools = "pan, wheel_zoom, xwheel_zoom, ywheel_zoom, reset, save" + + + # Tooltips and formatters + + dotplot_tooltips = [ + ("Name", "@mu_x"), + ("μ", "@mu{%0.18e}"), + ("Number of samples (tests)", "@nsamples") + ] + dotplot_formatters = { + "@mu" : "printf" + } + + sigma_boxplot_tooltips = self.gen_boxplot_tooltips("sigma") + sigma_boxplot_tooltips_formatters = self.gen_boxplot_tooltips_formatters("sigma") + + s10_boxplot_tooltips = self.gen_boxplot_tooltips("s10") + s10_boxplot_tooltips_formatters = self.gen_boxplot_tooltips_formatters("s10") + + s2_boxplot_tooltips = self.gen_boxplot_tooltips("s2") + s2_boxplot_tooltips_formatters = self.gen_boxplot_tooltips_formatters("s2") + + + # Plots + + # Mu plot + self.plots["mu_inspect"] = figure( + name="mu_inspect", + title="", + plot_width=900, plot_height=400, x_range=[""], + tools=tools, sizing_mode="scale_width" + ) + plot.fill_dotplot( + self.plots["mu_inspect"], self.sources["mu_source"], "mu", + tooltips = dotplot_tooltips, + tooltips_formatters = dotplot_formatters + ) + self.doc.add_root(self.plots["mu_inspect"]) + + + # Sigma plot + self.plots["sigma_inspect"] = figure( + name="sigma_inspect", + title="", + plot_width=900, plot_height=400, x_range=[""], + tools=tools, sizing_mode="scale_width" + ) + plot.fill_boxplot( + self.plots["sigma_inspect"], self.sources["sigma_source"], prefix="sigma", + tooltips = sigma_boxplot_tooltips, + tooltips_formatters = sigma_boxplot_tooltips_formatters + ) + self.doc.add_root(self.plots["sigma_inspect"]) + + + # s plots + self.plots["s10_inspect"] = figure( + name="s10_inspect", + title="", + plot_width=900, plot_height=400, x_range=[""], + tools=tools, sizing_mode='scale_width' + ) + plot.fill_boxplot( + self.plots["s10_inspect"], self.sources["s10_source"], prefix="s10", + tooltips = s10_boxplot_tooltips, + tooltips_formatters = s10_boxplot_tooltips_formatters + ) + s10_tab_inspect = Panel(child=self.plots["s10_inspect"], title="Base 10") + + self.plots["s2_inspect"] = figure( + name="s2_inspect", + title="", + plot_width=900, plot_height=400, x_range=[""], + tools=tools, sizing_mode='scale_width' + ) + plot.fill_boxplot( + self.plots["s2_inspect"], self.sources["s2_source"], prefix="s2", + tooltips = s2_boxplot_tooltips, + tooltips_formatters = s2_boxplot_tooltips_formatters + ) + s2_tab_inspect = Panel(child=self.plots["s2_inspect"], title="Base 2") + + s_tabs_inspect = Tabs( + name = "s_tabs_inspect", + tabs=[s10_tab_inspect, s2_tab_inspect], tabs_location = "below" + ) + self.doc.add_root(s_tabs_inspect) + + + + def setup_widgets(self): + + # Generation of selectable items + + # Dict contains all inspectable runs (maps display strings to timestamps) + # The dict structure allows to get the timestamp from the display string + # in O(1) + self.runs_dict = self.gen_runs_selection() + + # Dict maps display strings to column names for the different factors + # (var, backend, test) + self.factors_dict = { + "Variables": "variable", + "Backends": "vfc_backend", + "Tests": "test" + } + + + # Run selection + + # Contains all options strings + runs_display = list(self.runs_dict.keys()) + # Will be used when updating plots (contains actual number) + self.current_run = self.runs_dict[runs_display[-1]] + # Contains the selected option string, used to update current_n_runs + current_run_display = runs_display[-1] + # This contains only entries matching the run + self.run_data = self.data[self.data["timestamp"] == self.current_run] + + + change_run_callback_js="updateRunMetadata(cb_obj.value);" + + self.widgets["select_run"] = Select( + name="select_run", title="Run :", + value=current_run_display, options=runs_display + ) + self.doc.add_root(self.widgets["select_run"]) + self.widgets["select_run"].on_change("value", self.update_run) + self.widgets["select_run"].js_on_change("value", CustomJS( + code = change_run_callback_js, + args=(dict( + metadata=helper.metadata_to_dict( + helper.get_metadata(self.metadata, self.current_run) + ) + )) + )) + + + # Factors selection + + # "Group by" radio + self.widgets["groupby_radio"] = RadioButtonGroup( + name="groupby_radio", + labels=list(self.factors_dict.keys()), active=0 + ) + self.doc.add_root(self.widgets["groupby_radio"]) + # The functions are defined inside the template to avoid writing too + # much JS server side + self.widgets["groupby_radio"].on_change( + "active", + self.update_groupby + ) + + + # "Filter by" radio + # Get all possible factors, and remove the one selected in "Group by" + filterby_list = list(self.factors_dict.keys()) + del filterby_list[self.widgets["groupby_radio"].active] + + self.widgets["filterby_radio"] = RadioButtonGroup( + name="filterby_radio", + labels=filterby_list, active=0 + ) + self.doc.add_root(self.widgets["filterby_radio"]) + # The functions are defined inside the template to avoid writing too + # much JS server side + self.widgets["filterby_radio"].on_change( + "active", + self.update_filterby + ) + + + # Filter selector + + filterby = self.widgets["filterby_radio"].labels[ + self.widgets["filterby_radio"].active + ] + filterby = self.factors_dict[filterby] + + options = self.run_data.index\ + .get_level_values(filterby).drop_duplicates().tolist() + + self.widgets["select_filter"] = Select( + # We need a different name to avoid collision in the template with + # the runs comparison's widget + name="select_filter", title="Select a filter :", + value=options[0], options=options + ) + self.doc.add_root(self.widgets["select_filter"]) + self.widgets["select_filter"]\ + .on_change("value", self.update_filter) + + + # Toggle for outliers filtering + + self.widgets["outliers_filtering_inspect"] = CheckboxGroup( + name="outliers_filtering_inspect", + labels=["Filter outliers"], active = [] + ) + self.doc.add_root(self.widgets["outliers_filtering_inspect"]) + self.widgets["outliers_filtering_inspect"]\ + .on_change("active", self.update_outliers_filtering) + + + + # Communication methods + # (to send/receive messages to/from master) + + # When received, switch to the run_name in parameter + def switch_view(self, run_name): + self.widgets["select_run"].value = run_name + + + + # Constructor + + def __init__(self, master, doc, data, metadata): + + self.master = master + + self.doc = doc + self.data = data + self.metadata = metadata + + + self.sources = { + "mu_source": ColumnDataSource(data={}), + "sigma_source": ColumnDataSource(data={}), + "s10_source" :ColumnDataSource(data={}), + "s2_source": ColumnDataSource(data={}) + } + + self.plots = {} + self.widgets = {} + + # Setup Bokeh objects + self.setup_plots() + self.setup_widgets() + + # Pass the initial metadata to the template (will be updated in CustomJS + # callbacks). This is required because metadata is not displayed in a + # Bokeh widget, so we can't update this with a server callback. + initial_run = helper.get_metadata(self.metadata, self.current_run) + self.doc.template_variables["initial_timestamp"] = self.current_run + + # At this point, everything should have been initialized, so we can + # show the plots for the first time + self.update_plots() diff --git a/ci/vfc_ci_report/main.py b/ci/vfc_ci_report/main.py new file mode 100644 index 0000000..d011b3e --- /dev/null +++ b/ci/vfc_ci_report/main.py @@ -0,0 +1,217 @@ +# Look for and read all the run files in the current directory (ending with +# .vfcrun.hd5), and lanch a Bokeh server for the visualization of this data. + +import os +import sys +import time + +import pandas as pd + +from bokeh.plotting import curdoc + +# Local imports from vfc_ci_server +import compare_runs +import inspect_runs +import helper + +################################################################################ + + + # Read vfcrun files, and aggregate them in one dataset + +run_files = [ f for f in os.listdir(".") if f.endswith(".vfcrun.hd5") ] + +if len(run_files) == 0: + print( + "Warning [vfc_ci]: Could not find any vfcrun files in the directory. " \ + "This will result in server errors and prevent you from viewing the report." + ) + +# These are arrays of Pandas dataframes for now +metadata = [] +data = [] + +for f in run_files: + metadata.append(pd.read_hdf(f, "metadata")) + data.append(pd.read_hdf(f, "data")) + +metadata = pd.concat(metadata).sort_index() +data = pd.concat(data).sort_index() + + +# Generate the display strings for runs (runs ticks) +# By doing this in master, we ensure the homogeneity of display strings +# across all plots +metadata["name"] = metadata.index.to_series().map( + lambda x: helper.get_run_name( + x, + helper.get_metadata(metadata, x)["hash"] + ) +) +helper.reset_run_strings() + +metadata["date"] = metadata.index.to_series().map( + lambda x: time.ctime(x) +) + + +################################################################################ + + +curdoc().title = "Verificarlo Report" + + + # Read server arguments + # (this is quite easy because Bokeh server is called through a wrapper, so + # we know exactly what the arguments might be) + +git_repo_linked = False +commit_link = "" + +has_logo = False +logo_url = "" + +for i in range(1, len(sys.argv)): + + # Look for the Git repository remote address + # (if a Git repo is specified, the webpage will contain hyperlinks to the + # repository and the different commits) + if sys.argv[i] == "git": + from urllib.parse import urlparse + + method = sys.argv[i + 1] + address = sys.argv[i + 2] + url = "" + + + # Here, address is either the remote URL or the path to the local Git + # repo (depending on the method) + + if method == "url": + # We should directly have a Git URL + url = address + + elif method == "directory": + # Get the remote URL from the local repo + from git import Repo + repo = Repo(address) + url = repo.remotes.origin.url + + else: + raise ValueError( + "Error [vfc_ci]: The specified method to get the Git " \ + "repository is invalid. Are you calling Bokeh directly " \ + "instead of using the Verificarlo wrapper ?" + ) + + + # At this point, "url" should be set correctly, we can get the repo's + # URL and name, after making sure we're on a Git URL + + parsed_url = urlparse(url) + + path = parsed_url.path.split("/") + if len(path) < 3: + raise ValueError( + "Error [vfc_ci]: The found URL doesn't seem to be pointing " \ + "to a Git repository (path is too short)" + ) + + repo_name = path[2] + + curdoc().template_variables["repo_url"] = url + curdoc().template_variables["repo_name"] = repo_name + + + # We should have a "github.com" or a "*gitlab*" URL + + if parsed_url.netloc == "github.com": + commit_link = "https://%s%s/commit/" \ + % (parsed_url.netloc, parsed_url.path) + + curdoc().template_variables["commit_link"] = commit_link + curdoc().template_variables["git_host"] = "GitHub" + + # Used in Bokeh tooltips + commit_link = commit_link + "@hash" + + # We assume we have a GitLab URL + else: + commit_link = "https://%s%s/-/commit/" \ + % (parsed_url.netloc, parsed_url.path) + + curdoc().template_variables["commit_link"] = commit_link + curdoc().template_variables["git_host"] = "GitLab" + + # Used in Bokeh tooltips + commit_link = commit_link + "@hash" + + git_repo_linked = True + + + + # Look for a logo URL + # If a logo URL is specified, it will be included in the report's header + if sys.argv[i] == "logo": + curdoc().template_variables["logo_url"] = sys.argv[i + 1] + has_logo = True + + +# After the loop, we know if a repo has been linked, if we have a logo, ... +curdoc().template_variables["git_repo_linked"] = git_repo_linked +curdoc().template_variables["has_logo"] = has_logo + + +################################################################################ + + + # Setup report views + +# Define a ViewsMaster class to allow two-ways communication between views. +# This approach by classes allows us to have separate scopes for each view and +# will be useful if we want to add new views at some point in the future +# (instead of having n views with n-1 references each). + +class ViewsMaster: + + # Communication functions + + def go_to_inspect(self, run_name): + self.inspect.switch_view(run_name) + + + #Constructor + + def __init__(self, data, metadata, git_repo_linked, commit_link): + + self.data = data + self.metadata = metadata + self.git_repo_linked = git_repo_linked + self.commit_link = commit_link + + # Pass metadata to the template as a JSON string + curdoc().template_variables["metadata"] = self.metadata.to_json(orient="index") + + # Runs comparison + self.compare = compare_runs.CompareRuns( + master = self, + doc = curdoc(), + data = data, + metadata = metadata, + ) + + # Runs inspection + self.inspect = inspect_runs.InspectRuns( + master = self, + doc = curdoc(), + data = data, + metadata = metadata, + ) + + +views_master = ViewsMaster( + data = data, + metadata = metadata, + git_repo_linked = git_repo_linked, + commit_link = commit_link +) diff --git a/ci/vfc_ci_report/plot.py b/ci/vfc_ci_report/plot.py new file mode 100644 index 0000000..270266e --- /dev/null +++ b/ci/vfc_ci_report/plot.py @@ -0,0 +1,151 @@ +# General functions for filling plots with data in all report's views + +from bokeh.plotting import figure +from bokeh.models import HoverTool, TapTool, CustomJS + +from math import pi + + +def fill_dotplot( + plot, source, data_field, + tooltips=None, tooltips_formatters=None, + js_tap_callback=None, server_tap_callback=None, + lines=False, + lower_bound=False +): + + # (Optional) Tooltip and tooltip formatters + if tooltips != None: + hover = HoverTool(tooltips = tooltips, mode="vline", names=["circle"]) + + if tooltips_formatters != None: + hover.formatters = tooltips_formatters + + plot.add_tools(hover) + + + # (Optional) Add TapTool (for JS tap callback) + if js_tap_callback != None: + tap = TapTool(callback=CustomJS(code=js_tap_callback)) + plot.add_tools(tap) + + + # (Optional) Add segment to represent a lower bound + if lower_bound: + lower_segment = plot.segment( + x0="%s_x" % data_field, y0=data_field, + x1="%s_x" % data_field, y1="%s_lower_bound" % data_field, + source=source, line_color="black" + ) + + + # Draw dots (actually Bokeh circles) + circle = plot.circle( + name="circle", + x="%s_x" % data_field, y=data_field, source=source, size=12 + ) + + + # (Optional) Draw lines between dots + if lines: + line = plot.line(x="%s_x" % data_field, y=data_field, source=source) + + + # (Optional) Add server tap callback + if server_tap_callback != None: + circle.data_source.selected.on_change("indices", server_tap_callback) + + + # Plot appearance + plot.xgrid.grid_line_color = None + plot.ygrid.grid_line_color = None + + plot.yaxis[0].formatter.power_limit_high = 0 + plot.yaxis[0].formatter.power_limit_low = 0 + plot.yaxis[0].formatter.precision = 3 + + plot.xaxis[0].major_label_orientation = pi/8 + + + +def fill_boxplot( + plot, source, + prefix="", + tooltips=None, tooltips_formatters=None, + js_tap_callback=None, server_tap_callback=None, +): + + # (Optional) Tooltip and tooltip formatters + if tooltips != None: + hover = HoverTool(tooltips = tooltips, mode="vline", names=["full_box"]) + + if tooltips_formatters != None: + hover.formatters = tooltips_formatters + + plot.add_tools(hover) + + + # (Optional) Add TapTool (for JS tap callback) + if js_tap_callback != None: + tap = TapTool(callback=CustomJS(code=js_tap_callback)) + plot.add_tools(tap) + + + # Draw boxes (the prefix argument modifies the fields of ColumnDataSource + # that are used) + + if prefix != "": + prefix = "%s_" % prefix + + # Stems + top_stem = plot.segment( + x0="%sx" % prefix, y0="%smax" % prefix, + x1="%sx" % prefix, y1="%squantile75" % prefix, + source=source, line_color="black" + ) + bottom_stem = plot.segment( + x0="%sx" % prefix, y0="%smin" % prefix, + x1="%sx" % prefix, y1="%squantile25" % prefix, + source=source, line_color="black" + ) + + # Boxes + full_box = plot.vbar( + name="full_box", + x="%sx" % prefix, width=0.5, + top="%squantile75" % prefix, bottom="%squantile25" % prefix, + source=source, line_color="black" + ) + bottom_box = plot.vbar( + x="%sx" % prefix, width=0.5, + top="%squantile50" % prefix, bottom="%squantile25" % prefix, + source=source, line_color="black" + ) + + # Mu dot + mu_dot = plot.dot( + x="%sx" % prefix, y="%smu" % prefix, size=30, source=source, + color="black" + ) + + + # (Optional) Add server tap callback + if server_tap_callback != None: + top_stem.data_source.selected.on_change("indices", server_tap_callback) + bottom_stem.data_source.selected.on_change("indices", server_tap_callback) + + full_box.data_source.selected.on_change("indices", server_tap_callback) + bottom_box.data_source.selected.on_change("indices", server_tap_callback) + + mu_dot.data_source.selected.on_change("indices", server_tap_callback) + + + # Plot appearance + plot.xgrid.grid_line_color = None + plot.ygrid.grid_line_color = None + + plot.yaxis[0].formatter.power_limit_high = 0 + plot.yaxis[0].formatter.power_limit_low = 0 + plot.yaxis[0].formatter.precision = 3 + + plot.xaxis[0].major_label_orientation = pi/8 diff --git a/ci/vfc_ci_report/templates/index.html b/ci/vfc_ci_report/templates/index.html new file mode 100644 index 0000000..2ec7fc6 --- /dev/null +++ b/ci/vfc_ci_report/templates/index.html @@ -0,0 +1,480 @@ + + + + + Verificarlo Report + + + + + + + + + + {% extends base %} + + + + + {% block contents %} + + + + + + + +
+ + {% if has_logo %} + + {% else %} + + {% endif %} +
+ + + + + {% endblock %} + + + diff --git a/ci/workflow_templates/ci_README.j2.md b/ci/workflow_templates/ci_README.j2.md new file mode 100644 index 0000000..a41ca96 --- /dev/null +++ b/ci/workflow_templates/ci_README.j2.md @@ -0,0 +1,7 @@ +## Verificarlo CI : `{{dev_branch}}` + +You are on the `{{ci_branch}}` branch, which is automatically updated with the +[Verificarlo](https://github.com/verificarlo/verificarlo) test results from +`{{dev_branch}}` (in the `vfcruns` directory). + +You can start a Verificarlo CI server at anytime using the run files of this branch. diff --git a/ci/workflow_templates/gitlab-ci.j2.yml b/ci/workflow_templates/gitlab-ci.j2.yml new file mode 100644 index 0000000..4a4c83d --- /dev/null +++ b/ci/workflow_templates/gitlab-ci.j2.yml @@ -0,0 +1,42 @@ +# This workflow will be executed when {{dev_branch}} is updated: +# it will run the configured tests and upload the results on vfc_ci_master. + +image: verificarlo/verificarlo + + +stages: + - run_verificarlo_tests + + +run_verificarlo_tests: + stage: run_verificarlo_tests + + before_script: + - git remote set-url origin https://{{username}}:${CI_PUSH_TOKEN}@{{remote_url}}.git + - git config --global user.email "{{email}}" + - git config --global user.name "{{username}}" + + script: + # We will probably drop these installations when integrating CI into + # Verificarlo + - pip install numpy scipy pandas bokeh jinja2 tables GitPython + - apt update + - apt install wget + - wget https://raw.githubusercontent.com/verificarlo/significantdigits/main/sigdigits.py -P /usr/local/lib/python3.8/dist-packages + + - ./vfc_ci test -g -r + - git_hash=$(git rev-parse --short "$CI_COMMIT_SHA") + - git fetch --all + - git checkout -b {{ci_branch}} origin/{{ci_branch}} + - mkdir -p vfcruns + - mv *.vfcrun.hd5 vfcruns + - git add vfcruns/* + - git commit -m "[auto] New test results for commit ${git_hash}" + - git push + + rules: + - if: '$CI_COMMIT_BRANCH == "{{dev_branch}}"' + + artifacts: + paths: + - "*.vfcraw.hd5" diff --git a/ci/workflow_templates/vfc_test_workflow.j2.yml b/ci/workflow_templates/vfc_test_workflow.j2.yml new file mode 100644 index 0000000..f332226 --- /dev/null +++ b/ci/workflow_templates/vfc_test_workflow.j2.yml @@ -0,0 +1,57 @@ +# This workflow will be executed when {{dev_branch}} is updated: +# it will run the configured tests and upload the results on {{ci_branch}}. + +name: "Verificarlo CI ({{dev_branch}})" + +on: + # Triggers the workflow when {{dev_branch}} is updated + push: + branches: [ {{dev_branch}} ] + + workflow_dispatch: + + +jobs: + run_verificarlo_tests: + runs-on: ubuntu-latest + container: verificarlo/verificarlo + + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + + # We will probably drop these installations when integrating CI into + # Verificarlo + - name: Install Python requirements + run: | + pip install numpy scipy pandas bokeh jinja2 tables GitPython + + apt install wget + wget https://raw.githubusercontent.com/verificarlo/significantdigits/main/sigdigits.py -P /usr/local/lib/python3.8/dist-packages + + - name: Run tests + # We assume the script is included in the repo for now + # (we'll probably want to remove "./" if the script ends up being integrated + # in Verificarlo and becomes available system-wide) + run: ./vfc_ci test -g -r + + - name: Commit test results + run: | + git_hash=$(git rev-parse --short "$GITHUB_SHA") + + git config --local user.email "action@github.com" + git config --local user.name "GitHub Action" + + git checkout {{ci_branch}} + mkdir -p vfcruns + mv *.vfcrun.hd5 vfcruns + git add vfcruns/* + git commit -m "[auto] New test results for commit ${git_hash}" + git push + + - name: Upload raw results as artifacts + uses: actions/upload-artifact@v2 + with: + {% raw %}name: ${{github.sha}}.vfcraw{% endraw %} + path: ./*.vfcraw.hd5 diff --git a/include/vfc_hashmap.h b/include/vfc_hashmap.h new file mode 100644 index 0000000..c314f0e --- /dev/null +++ b/include/vfc_hashmap.h @@ -0,0 +1,248 @@ +/* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +/* + * Copyright 2012 Couchbase, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#define HASH_MULTIPLIER 31 +static const unsigned int hashmap_prime_1 = 73; +static const unsigned int hashmap_prime_2 = 5009; + +#ifndef __VFC_HASHMAP_HEADER__ + +struct vfc_hashmap_st { + size_t nbits; + size_t mask; + + size_t capacity; + size_t *items; + size_t nitems; + size_t n_deleted_items; +}; +typedef struct vfc_hashmap_st *vfc_hashmap_t; + +// allocate and initialize the map +vfc_hashmap_t vfc_hashmap_create(); + +// get the value at an index of a map +size_t get_value_at(size_t *items, size_t i); + +// get the key at an index of a map +size_t get_key_at(size_t *items, size_t i); + +// set the value at an index of a map +void set_value_at(size_t *items, size_t value, size_t i); + +// set the key at an index of a map +void set_key_at(size_t *items, size_t key, size_t i); + +// free the map +void vfc_hashmap_destroy(vfc_hashmap_t map); + +// insert an element in the map +void vfc_hashmap_insert(vfc_hashmap_t map, size_t key, void *item); + +// remove an element of the map +void vfc_hashmap_remove(vfc_hashmap_t map, size_t key); + +// test if an element is in the map +char vfc_hashmap_have(vfc_hashmap_t map, size_t key); + +// get an element of the map +void *vfc_hashmap_get(vfc_hashmap_t map, size_t key); + +// get the number of elements in the map +size_t vfc_hashmap_num_items(vfc_hashmap_t map); + +// Hash function +size_t vfc_hashmap_str_function(const char *id); + +#endif + +/***************** Verificarlo hashmap FUNCTIONS ******************** + * The following set of functions are used in backends and wrapper + * to stock and access quickly internal data. + *******************************************************************/ + +// free the map +void vfc_hashmap_destroy(vfc_hashmap_t map) { + if (map) { + free(map->items); + } + free(map); +} + +// allocate and initialize the map +vfc_hashmap_t vfc_hashmap_create() { + vfc_hashmap_t map = (vfc_hashmap_t) calloc(1, sizeof(struct vfc_hashmap_st)); + + if (map == NULL) { + return NULL; + } + map->nbits = 3; + map->capacity = (size_t)(1 << map->nbits); + map->mask = map->capacity - 1; + // an item is now a value and a key + map->items = (size_t *) calloc(map->capacity, 2 * sizeof(size_t)); + if (map->items == NULL) { + vfc_hashmap_destroy(map); + return NULL; + } + map->nitems = 0; + map->n_deleted_items = 0; + return map; +} + +size_t get_value_at(size_t *items, size_t i) { return items[i * 2]; } + +size_t get_key_at(size_t *items, size_t i) { return items[(i * 2) + 1]; } + +void set_value_at(size_t *items, size_t value, size_t i) { + items[i * 2] = value; +} + +void set_key_at(size_t *items, size_t key, size_t i) { + items[(i * 2) + 1] = key; +} + +// add a member in the table +static int hashmap_add_member(vfc_hashmap_t map, size_t key, void *item) { + size_t value = (size_t)item; + size_t ii; + + if (value == 0 || value == 1) { + return -1; + } + + ii = map->mask & (hashmap_prime_1 * key); + + while (get_value_at(map->items, ii) != 0 && + get_value_at(map->items, ii) != 1) { + if (get_value_at(map->items, ii) == value) { + return 0; + } else { + /* search free slot */ + ii = map->mask & (ii + hashmap_prime_2); + } + } + map->nitems++; + if (get_value_at(map->items, ii) == 1) { + map->n_deleted_items--; + } + + set_value_at(map->items, value, ii); + set_key_at(map->items, key, ii); + + return 1; +} + +// rehash the table if necessary +static void maybe_rehash_map(vfc_hashmap_t map) { + size_t *old_items; + size_t old_capacity, ii; + + if (map->nitems + map->n_deleted_items >= (double)map->capacity * 0.85) { + old_items = map->items; + old_capacity = map->capacity; + map->nbits++; + map->capacity = (size_t)(1 << map->nbits); + map->mask = map->capacity - 1; + map->items = (size_t *) calloc(map->capacity, 2 * sizeof(size_t)); + map->nitems = 0; + map->n_deleted_items = 0; + for (ii = 0; ii < old_capacity; ii++) { + hashmap_add_member(map, get_key_at(old_items, ii), + (void *)get_value_at(old_items, ii)); + } + free(old_items); + } +} + +// insert an element in the map +void vfc_hashmap_insert(vfc_hashmap_t map, size_t key, void *item) { + hashmap_add_member(map, key, item); + maybe_rehash_map(map); +} + +// remove an element of the map +void vfc_hashmap_remove(vfc_hashmap_t map, size_t key) { + size_t ii = map->mask & (hashmap_prime_1 * key); + + while (get_value_at(map->items, ii) != 0) { + if (get_key_at(map->items, ii) == key) { + set_value_at(map->items, 1, ii); + map->nitems--; + map->n_deleted_items++; + break; + } else { + ii = map->mask & (ii + hashmap_prime_2); + } + } +} + +// test if an element is in the map +char vfc_hashmap_have(vfc_hashmap_t map, size_t key) { + size_t ii = map->mask & (hashmap_prime_1 * key); + + while (get_value_at(map->items, ii) != 0) { + if (get_key_at(map->items, ii) == key) { + return 1; + } else { + ii = map->mask & (ii + hashmap_prime_2); + } + } + return 0; +} + +// get an element of the map +void *vfc_hashmap_get(vfc_hashmap_t map, size_t key) { + size_t ii = map->mask & (hashmap_prime_1 * key); + + while (get_value_at(map->items, ii) != 0) { + if (get_key_at(map->items, ii) == key) { + return (void *)get_value_at(map->items, ii); + } else { + ii = map->mask & (ii + hashmap_prime_2); + } + } + return NULL; +} + +// get the number of elements in the map +size_t vfc_hashmap_num_items(vfc_hashmap_t map) { return map->nitems; } + +// Hash function for strings +size_t vfc_hashmap_str_function(const char *id) { + unsigned const char *us; + + us = (unsigned const char *)id; + + size_t index = 0; + + while (*us != '\0') { + index = index * HASH_MULTIPLIER + *us; + us++; + } + + return index; +} + +// Free the hashmap +void vfc_hashmap_free(vfc_hashmap_t map) { + for (int ii = 0; ii < map->capacity; ii++) + if (get_value_at(map->items, ii) != 0 && get_value_at(map->items, ii) != 0) + free((void *)get_value_at(map->items, ii)); +} diff --git a/include/vfc_probe.h b/include/vfc_probe.h new file mode 100644 index 0000000..0d716d1 --- /dev/null +++ b/include/vfc_probe.h @@ -0,0 +1,254 @@ +/* +* This file defines "vfc_probes", a hashtable-based structure which can be used +* to place "probes" in a code and store the different values of test variables. +* These test results can then be exported in a CSV file, and used to generate a +* Verificarlo test report. +*/ + + +#include +#include +#include + +#include "vfc_hashmap.h" + +#define VAR_NAME(var) #var // Simply returns the name of var into a string + + +/* +* A probe containing a double value as well as its key, which is needed when +* dumping the probes +*/ + +struct vfc_probe_node { + char * key; + double value; +}; + +typedef struct vfc_probe_node vfc_probe_node; + + + +/* +* The probes structure. It simply acts as a wrapper for a Verificarlo hashmap. +*/ + +struct vfc_probes { + vfc_hashmap_t map; +}; + +typedef struct vfc_probes vfc_probes; + + +/* +* Initialize an empty vfc_probes instance +*/ + +vfc_probes vfc_init_probes() { + vfc_probes probes; + probes.map = vfc_hashmap_create(); + + return probes; +} + + + +/* +* Free all probes +*/ + +void vfc_free_probes(vfc_probes * probes) { + + // Before freeing the map, iterate manually over all items to free the keys + vfc_probe_node * probe = NULL; + for(int i = 0; i < probes->map->capacity; i++) { + probe = (vfc_probe_node*) get_value_at(probes->map->items, i); + if(probe != NULL) { + if(probe->key != NULL) { + free(probe->key); + } + } + } + + vfc_hashmap_free(probes->map); +} + + + +/* +* Helper function to generate the key from test and variable name +*/ + +char * gen_probe_key(char * testName, char * varName) { + char * key = (char *) malloc(strlen(testName) + strlen(varName) + 2); + strcpy(key, testName); + strcat(key, ","); + strcat(key, varName); + + return key; +} + + + +/* +* Helper function to detect forbidden character ',' in the keys +*/ + +void validate_probe_key(char * str) { + unsigned int len = strlen(str); + + for(unsigned int i=0; imap, vfc_hashmap_str_function(key) + ); + + if(oldProbe != NULL) { + if(strcmp(key, oldProbe->key) == 0) { + fprintf( + stderr, + "Error [verificarlo]: you have a duplicate error with one of \ + your probes (\"%s\"). Please make sure to use different names.\n", + key + ); + exit(1); + } + } + + // Insert the element in the hashmap + vfc_probe_node * newProbe = (vfc_probe_node*) malloc(sizeof(vfc_probe_node)); + newProbe->key = key; + newProbe->value = val; + + vfc_hashmap_insert( + probes->map, vfc_hashmap_str_function(key), newProbe + ); + + return 0; +} + + + +/* +* Remove (free) an element from the hash table +*/ + +int vfc_remove_probe(vfc_probes * probes, char * testName, char * varName) { + + if(probes == NULL) { + return 1; + } + + // Get the key, which is : testName + "," + varName + char * key = gen_probe_key(testName, varName); + + vfc_hashmap_remove(probes->map, vfc_hashmap_str_function(key)); + + return 0; +} + + + +/* +* Return the number of probes stored in the hashmap +*/ + +unsigned int vfc_num_probes(vfc_probes * probes) { + return vfc_hashmap_num_items(probes->map); +} + + + +/* +* Dump probes in a .csv file (the double values are converted to hex), then +* free it. +*/ + +int vfc_dump_probes(vfc_probes * probes) { + + if(probes == NULL) { + return 1; + } + + // Get export path from the VFC_PROBES_OUTPUT env variable + char* exportPath = getenv("VFC_PROBES_OUTPUT"); + if(!exportPath) { + printf( + "Warning [verificarlo]: VFC_PROBES_OUTPUT is not set, probes will \ + not be dumped\n" + ); + vfc_free_probes(probes); + return 0; + } + + FILE * fp = fopen(exportPath, "w"); + + if(fp == NULL) { + fprintf( + stderr, + "Error [verificarlo]: impossible to open the CSV file to save your \ + probes (\"%s\")\n", + exportPath + ); + exit(1); + } + + // First line gives the column names + fprintf(fp, "test,variable,value\n"); + + // Iterate over all table elements + vfc_probe_node * probe = NULL; + for(int i = 0; i < probes->map->capacity; i++) { + probe = (vfc_probe_node*) get_value_at(probes->map->items, i); + if(probe != NULL) { + fprintf( + fp, "%s,%a\n", + probe->key, + probe->value + ); + } + } + + fclose(fp); + + vfc_free_probes(probes); + + return 0; +} diff --git a/tests/vfc_test_h5.cpp b/tests/vfc_test_h5.cpp new file mode 100644 index 0000000..18b0435 --- /dev/null +++ b/tests/vfc_test_h5.cpp @@ -0,0 +1,196 @@ +// This files is almost the same as test_h5.cpp, with the difference that it +// dumps Verificarlo probes for vfc_ci integration, and that it reads a list of +// cycles in a CSV file, instead of accepting a start and an end cycle (which +// makes it easier to select the exact cycles we are interested in with vfc_ci). + +#include +#include + +#include +#include +#include + + +#include "SM_MaponiA3.hpp" +#include "SM_Standard.hpp" +#include "SM_Helpers.hpp" +#include "vfc_probe.h" + +using namespace H5; +// #define DEBUG + +const H5std_string FILE_NAME( "datasets/dataset.hdf5" ); + +double residual_max(double * A, unsigned int Dim) { + double max = 0.0; + for (unsigned int i = 0; i < Dim; i++) { + for (unsigned int j = 0; j < Dim; j++) { + double delta = (A[i * Dim + j] - (i == j)); + delta = abs(delta); + if (delta > max) max = delta; + } + } + return max; +} + +double residual2(double * A, unsigned int Dim) { + double res = 0.0; + for (unsigned int i = 0; i < Dim; i++) { + for (unsigned int j = 0; j < Dim; j++) { + double delta = (A[i * Dim + j] - (i == j)); + res += delta*delta; + } + } + return res; +} + +void read_int(H5File file, std::string key, unsigned int * data) { + DataSet ds = file.openDataSet(key); + ds.read(data, PredType::STD_U32LE); + ds.close(); +} + +void read_double(H5File file, std::string key, double * data) { + DataSet ds = file.openDataSet(key); + ds.read(data, PredType::IEEE_F64LE); + ds.close(); +} + + +/* Return a vector containing all cycles to execute by reading a data file */ +std::vector get_cycles_list(std::string path) { + std::ifstream file_stream(path); + std::stringstream string_stream; + string_stream << file_stream.rdbuf(); + + std::string cycle_str; + std::vector cycles_list = {}; + + while(string_stream >> cycle_str) { + cycles_list.push_back(std::stoi(cycle_str)); + } + + return cycles_list; +} + +int test_cycle(H5File file, int cycle, std::string version, vfc_probes * probes) { + + /* Read the data */ + + std::string group = "cycle_" + std::to_string(cycle); + + try{ + file.openGroup(group); + } catch(H5::Exception& e){ + std::cerr << "group " << group << "not found" << std::endl; + return 0; + } + + unsigned int dim, nupdates, col, i, j; + read_int(file, group + "/slater_matrix_dim", &dim); + read_int(file, group + "/nupdates", &nupdates); + + + double * slater_matrix = new double[dim*dim]; + read_double(file, group + "/slater_matrix", slater_matrix); + + double * slater_inverse = new double[dim*dim]; + read_double(file, group + "/slater_inverse", slater_inverse); + //slater_inverse = transpose(slater_inverse, dim); + + unsigned int * col_update_index = new unsigned int[nupdates]; + read_int(file, group + "/col_update_index", col_update_index); + + double * updates = new double[nupdates*dim]; + read_double(file, group + "/updates", updates); + + double * u = new double[nupdates*dim]; + + /* Test */ +#ifdef DEBUG + showMatrix(slater_matrix, dim, "OLD Slater"); +#endif + +#ifdef DEBUG + showMatrix(slater_inverse, dim, "OLD Inverse"); +#endif + + for (j = 0; j < nupdates; j++) { + for (i = 0; i < dim; i++) { + col = col_update_index[j]; + u[i + j*dim] = updates[i + j*dim] - slater_matrix[i*dim + (col - 1)]; + slater_matrix[i*dim + (col - 1)] = updates[i + j*dim]; + } + } + + if (version == "maponia3") { + MaponiA3(slater_inverse, dim, nupdates, u, col_update_index); + } else if (version == "sm1") { + SM1(slater_inverse, dim, nupdates, u, col_update_index); + } else if (version == "sm2") { + SM2(slater_inverse, dim, nupdates, u, col_update_index); + } else if (version == "sm3") { + SM3(slater_inverse, dim, nupdates, u, col_update_index); + } else { + std::cerr << "Unknown version " << version << std::endl; + exit(1); + } + +#ifdef DEBUG + showMatrix(slater_matrix, dim, "NEW Slater"); +#endif + +#ifdef DEBUG + showMatrix(slater_inverse, dim, "NEW Inverse"); +#endif + + double * res = new double[dim*dim] {0}; + matMul(slater_matrix, slater_inverse, res, dim); + bool ok = is_identity(res, dim, 1e-3); + + double res_max = residual_max(res, dim); + double res2 = residual2(res, dim); + +#ifdef DEBUG + showMatrix(res, dim, "Result"); +#endif + + vfc_put_probe(probes, &(group)[0], &("res_max_" + version)[0], res_max); + vfc_put_probe(probes, &(group)[0], &("res2_" + version)[0], res2); + + delete [] res, updates, u, col_update_index, + slater_matrix, slater_inverse; + + return ok; +} + +int main(int argc, char **argv) { + if (argc != 3) { + std::cerr << "Execute from within '/'" << std::endl; + std::cerr << "usage: test_h5 " << std::endl; + return 1; + } + std::string version(argv[1]); + std::vector cycles_list = get_cycles_list(argv[2]); + H5File file(FILE_NAME, H5F_ACC_RDONLY); + + vfc_probes probes = vfc_init_probes(); + probes = vfc_init_probes(); + + bool ok; + for (int i = 0; i < cycles_list.size(); i++) { + ok = test_cycle(file, cycles_list[i], version, &probes); + if (ok) { + std::cout << "ok -- cycle " << std::to_string(i) + << std::endl; + } + else { + std::cerr << "failed -- cycle " << std::to_string(i) + << std::endl; + } + } + + vfc_dump_probes(&probes); + + return ok; +} diff --git a/vfc_ci b/vfc_ci new file mode 100755 index 0000000..c695834 --- /dev/null +++ b/vfc_ci @@ -0,0 +1,205 @@ +#!/usr/bin/env python3 + +# This is the entry point of the Verificarlo CI command line interface, which is +# based on argparse and this article : +# https://mike.depalatis.net/blog/simplifying-argparse.html +# From here, 3 subcommands can be called : +# - setup : create a vfc_ci branch and workflow on the current Git repo +# - test : run and export test results according to the vfc_tests_config.json +# - serve : launch a Bokeh server to visualize run results + +import argparse + +################################################################################ + + # Parameters validation helpers + +def is_port(string): + value = int(string) + if value < 0 or value > 65535: + raise argparse.ArgumentTypeError("Value has to be between 0 and 65535") + return value + + +def is_directory(string): + import os + + isdir = os.path.isdir(string) + if not isdir: + raise argparse.ArgumentTypeError("Directory does not exist") + + return string + + +################################################################################ + + + # Subcommand decorator + +cli = argparse.ArgumentParser( + description="Define, run, automatize, and visualize custom Verificarlo tests." +) +subparsers = cli.add_subparsers(dest="subcommand") + +def subcommand(description="", args=[], parent=subparsers): + def decorator(func): + parser = parent.add_parser(func.__name__, description=description) + for arg in args: + parser.add_argument(*arg[0], **arg[1]) + parser.set_defaults(func=func) + return decorator + + +def argument(*name_or_flags, **kwargs): + return ([*name_or_flags], kwargs) + + + +################################################################################ + + + # "setup" subcommand + +@subcommand( + description="Create an automated workflow to execute Verificarlo tests.", + args = [ + argument( + "git_host", + help=""" + specify where your repository is hosted + """, + choices=["github", "gitlab"] + ) + ] +) +def setup(args): + import ci.setup + ci.setup.setup(args.git_host) + + + + # "test" subcommand + +@subcommand( + description="Execute predefined Verificarlo tests and save their results.", + args = [ + argument( + "-g", "--is-git-commit", + help=""" + When specified, the last Git commit of the local repository (working + directory) will be fetched and associated with the run. + """, + action="store_true" + ), + argument( + "-r", "--export-raw-results", + help=""" + Specify if an additional HDF5 file containing the raw results must be + exported. + """, + action="store_true" + ), + argument( + "-d", "--dry-run", + help=""" + Perform a dry run by not saving the test results. + """, + action="store_true" + ) + ] +) +def test(args): + import ci.test + ci.test.run(args.is_git_commit, args.export_raw_results, args.dry_run) + + + + # "serve" subcommand + +@subcommand( + description=""" + Start a server to visualize Verificarlo test results. + """, + args = [ + argument( + "-s", "--show", + help=""" + Specify if the report must be opened in the browser at server + startup. + """, + action="store_true" + ), + argument( + "-gd", "--git-directory", + help=""" + Path to a local Git repository. The report will be linked to the + remote URL (GitHub and GitLab are supported). + """, + type=is_directory + ), + argument( + "-gu", "--git-url", + help=""" + GitHub or GitLab repository URL. The report will be linked to this + URL. + """, + type=str + ), + argument( + "-p", "--port", + help=""" + The port on which the server will run. Defaults to 8080. + """, + type=is_port, + default=8080 + ), + argument( + "-a", "--allow-origin", + help=""" + The origin (URL) from which the report will be accessible. + Port number must not be specified. Defaults to * (allow everything). + """, + type=str, + default="*" + ), + argument( + "-l", "--logo", + help=""" + Specify the URL of an image to be displayed in the report header. + """, + type=str + ) + ] +) +def serve(args): + + # git_directory and git_url are supposed to be exclusive + if args.git_directory != None and args.git_url != None: + raise argparse.ArgumentTypeError( + "\"-gd\" / \"--git-directory\" and \"-gu\" / \"--git-url\" are "\ + "mutually exclusive. Please make sure to use at most one of them." + ) + + import ci.serve + ci.serve.serve( + args.show, + args.git_directory, + args.git_url, + args.port, + args.allow_origin, + args.logo + ) + + + +############################################################################### + + + # Main command group and entry point + +if __name__ == "__main__": + args = cli.parse_args() + if args.subcommand is None: + cli.print_help() + else: + args.func(args) diff --git a/vfc_ci_cycles.txt b/vfc_ci_cycles.txt new file mode 100644 index 0000000..840505a --- /dev/null +++ b/vfc_ci_cycles.txt @@ -0,0 +1,2 @@ +1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 +327 655 983 1311 1639 1967 2295 2623 2951 3279 3607 3935 4263 4591 4919 5247 5575 5903 6231 6559 6887 7215 7543 7871 8199 diff --git a/vfc_tests_config.json b/vfc_tests_config.json new file mode 100644 index 0000000..abae182 --- /dev/null +++ b/vfc_tests_config.json @@ -0,0 +1,92 @@ +{ + "make_command": "make -f Makefile.vfc_ci", + "executables": [ + { + "executable": "bin/vfc_test_h5", + "parameters" : "maponia3 vfc_ci_cycles.txt", + "vfc_backends": [ + { + "name": "libinterflop_mca.so", + "repetitions": 50 + } + ] + }, + + { + "executable": "bin/vfc_test_h5", + "parameters" : "sm1 vfc_ci_cycles.txt", + "vfc_backends": [ + { + "name": "libinterflop_mca.so", + "repetitions": 50 + } + ] + }, + + { + "executable": "bin/vfc_test_h5", + "parameters" : "sm2 vfc_ci_cycles.txt", + "vfc_backends": [ + { + "name": "libinterflop_mca.so", + "repetitions": 50 + } + ] + }, + + { + "executable": "bin/vfc_test_h5", + "parameters" : "sm3 vfc_ci_cycles.txt", + "vfc_backends": [ + { + "name": "libinterflop_mca.so", + "repetitions": 50 + } + ] + }, + + { + "executable": "bin/vfc_test_h5", + "parameters" : "maponia3 vfc_ci_cycles.txt", + "vfc_backends": [ + { + "name": "libinterflop_mca.so --mode=rr", + "repetitions": 50 + } + ] + }, + + { + "executable": "bin/vfc_test_h5", + "parameters" : "sm1 vfc_ci_cycles.txt", + "vfc_backends": [ + { + "name": "libinterflop_mca.so --mode=rr", + "repetitions": 50 + } + ] + }, + + { + "executable": "bin/vfc_test_h5", + "parameters" : "sm2 vfc_ci_cycles.txt", + "vfc_backends": [ + { + "name": "libinterflop_mca.so --mode=rr", + "repetitions": 50 + } + ] + }, + + { + "executable": "bin/vfc_test_h5", + "parameters" : "sm3 vfc_ci_cycles.txt", + "vfc_backends": [ + { + "name": "libinterflop_mca.so --mode=rr", + "repetitions": 50 + } + ] + } + ] +}