From 44f0fc1f51b97a44902f46244be655c7955eb9bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Delval?= Date: Mon, 3 May 2021 15:47:44 +0200 Subject: [PATCH] Update vfc_ci code --- ci/__pycache__/__init__.cpython-39.pyc | Bin 154 -> 154 bytes ci/__pycache__/serve.cpython-39.pyc | Bin 656 -> 656 bytes ci/__pycache__/setup.cpython-39.pyc | Bin 3296 -> 3309 bytes ci/__pycache__/test.cpython-39.pyc | Bin 7108 -> 7071 bytes ci/serve.py | 10 +- ci/setup.py | 67 +++--- ci/test.py | 178 +++++++------- .../__pycache__/compare_runs.cpython-39.pyc | Bin 9285 -> 9336 bytes .../__pycache__/helper.cpython-39.pyc | Bin 2839 -> 2844 bytes .../__pycache__/inspect_runs.cpython-39.pyc | Bin 10125 -> 10165 bytes .../__pycache__/plot.cpython-39.pyc | Bin 2455 -> 2448 bytes ci/vfc_ci_report/compare_runs.py | 220 ++++++++---------- ci/vfc_ci_report/helper.py | 24 +- ci/vfc_ci_report/inspect_runs.py | 208 ++++++++--------- ci/vfc_ci_report/main.py | 75 +++--- ci/vfc_ci_report/plot.py | 43 ++-- ci/vfc_ci_report/templates/index.html | 8 +- ci/workflow_templates/gitlab-ci.j2.yml | 4 +- .../vfc_test_workflow.j2.yml | 6 +- vfc_ci | 32 ++- 20 files changed, 398 insertions(+), 477 deletions(-) diff --git a/ci/__pycache__/__init__.cpython-39.pyc b/ci/__pycache__/__init__.cpython-39.pyc index 0135b7144a5614d2edd969d33d163e71293481c8..18acc29125bbcc374f0eacb0949d9c8f051260e3 100644 GIT binary patch delta 19 ZcmbQmIE#@xk(ZZ?0SF#_>z~No4*)DN1z7+9 delta 19 ZcmbQmIE#@xk(ZZ?0SGQ9cTVK)2LLGG1hD`B diff --git a/ci/__pycache__/serve.cpython-39.pyc b/ci/__pycache__/serve.cpython-39.pyc index b39aba02ae1a52a39f4b57bb54ccd28adafb97bf..aac8c1020fe7d167601c159ddf3b67dadd1ff46b 100644 GIT binary patch delta 103 zcmbQhI)Rlhk(ZZ?0SILO_9rkiPvpx}D`hNFNMWvF6l17mt6@rE0P+n|SZbI+{1m2I o_8L}@*v3WujPmS2jm0t`?aVw(EQ}lw2w^etFi#F(vS(xl06IVsWdHyG delta 103 zcmbQhI)Rlhk(ZZ?0SFi?x)MG!P2|f{%VsW8NMWvF6l17mt6@rE0P+n|SZbI+{1m2I o_8L}@*v3WujPh(ijm0t`?aVw(EQ}lw2w^etuuKkMvS(xh07|(Ll>h($ diff --git a/ci/__pycache__/setup.cpython-39.pyc b/ci/__pycache__/setup.cpython-39.pyc index a6aeba4f88a4a1f4ee501b7334633af89fe7d7f4..f6d3998e0829c5fbd8d9ab5e1374024bdcdfff3b 100644 GIT binary patch delta 125 zcmaDL`BsuQk(ZZ?0SGE{yArN#;J zICt}6mR~H4vnS8v)D%|&sVPYe+%Q%1AR z>sWrVFwUI3l2cQ(EG;>{B(=DtI6gT)FD)}&FRM5|Z}Lk{4PJGSRC1;QNLGEb92XxG Mqwi#0p8bsa0OGGB<^TWy diff --git a/ci/__pycache__/test.cpython-39.pyc b/ci/__pycache__/test.cpython-39.pyc index fa73d452612722f74b4fa60f0eb72f15333d46a3..f7a0037aff6919223e9dc5a48e1291c8412e966e 100644 GIT binary patch delta 1431 zcmZ8hO>7%Q6!z?{*Xwos)7XUAX&|Yaq?@Lp39U+1leDx|Lm(k>C`Ajx*?2ehrvA&D zb^eqR!HG)+(j2%TLB$Dip;+Psf&;gL#0jJoYLBQ0ap(aF2?X!0iv-MSzMYx(&3p5{ zx3gcLdH+n@ipTX3wmT2rUM=2^w`6usye^NcTkGXg+3kEGTP&fVUB!M8I|nfl9z56& zZOEQ-Jp?a$;SW@tMJntTyCHenMs%O;(_7z&A@*#iN!AY+*O48f56XbYUO%iU(fSL=>fZQ4MVf^i7wM1BMuUbR{dWND_h zwyav)@%6gfC_NlYjg}v_0IXeKs<(ZK+hRc-J}5dh%X7CJUtI{?nvp9`!vU*iDz#>1 z+SIHy-mJAf#}9M2Qnx-*Ab}yw9^2&}xNB_>&jjNXm2pVoy9+5AO&~mvo%bVz6rH7H zT}ns_lkicQ`iGj=G$o*wDV={B39VrdgkBR$WVv7~cKD_m$fuHgfk=_ANN@zXm2bbH z_RDN#J!rqq8Vcmm07u2W@X_Z+X*h;}sZR%@!3ZspU~r7UAfQa(H~}3VP7q8IJb_@! z7Z>MXO8gL+Rx3E?hecBBk4l)%@2U=`Iulx*W#1wTi4?*td)T;?V5*dpV5akFG|z_5 zqv3u80iHqFi;VE0!U&%MO4esOToS+OC)s5&78@O>H8bqgEw=`j$R6gcn(G;1xIi-v zKLyLSJPR(1?buW7UGYim1Y7Fdjin_vD;~t(V9UZw++_=5CApL>khe&qwC>g%4v*ms zmI;dD^W>4}Rr1JOI46EePO;BKHg%1yb~aPDnc9^{9P2fhDvG@mYM|0dv6;$4LUhKHE|RU#;(!Jr`C%jB6NzRDP^+W9$C zVyM7;-}tL9A<2-t)qaFFn6w~uRVdNIf4^Nn&a2Js(zXQ+w^89^m=~wKLRe8O_g{+I z%X*{?%P?>|-}T*LXU~(9prraF{x!hbo~)h*(%fLaeBIfF7l=XEb+5~4H;}P6FTU@e zQ%+CrT^0H4irCJ+GBrcQC4yA~m!OJZM!Dma+=lJ!m^yF5!!axfIhSV_#8_@9_8O6c zNCi>Mjk2f3Zf=Un;^W-lqjj_yS$LT!X9=Dqm>2hQ(+7d0J23*NZRsaA1 delta 1424 zcmZuwO>7%g5cb=D+p(S6sbz;IAxRThlcY`Bs1-B~rKChHiJAo3;vU?McjK&Muibg; zkVFm?plW&J3pD_j*Sx}74K`XI$`-rxTZ}bD zpW3(Q%kL@8P+2Xp_t~z5C(YM4La^)`<#0K&E8~B($(j=Wca>vJ*_LlByNWjy*yI0W zS8T;rZ)>|sQ?a#dXeDIoiy&il4;jJ-QafA+r|5FG=C!q|xTmVg(!d)(E#2?`an@Ae zC_2H2nA49AYItnx;P7>;?m(6#$*Z?j3&7fLOVx(>PEDt{-PRTyf4yes^%oq^0jnZ@ zRR{OZt@2u>;X7^l{NlW+LY(5eY?u4st~PwU9?ak_(lZVT5(4O^(*(kc_~SW*kP>0t za$JrugDFhHug0WDM&uuogxcS=Xb@M#PKBnXSzg@NmiBO9G7e4&hM{(b0YjJ-h6!ja zW&-FQcnYB{@hPC*VU&RSh9?M~BnWCAqZ3sK#|VxSj1x>E;1H=-m?A)TIRq!f&2&n9 zsSjv?N}drv>lyWtro(hAW>i`BZPHRu2wUr7IVsLGsaJy8mIx2C!E}E@x*bn~sa?RvZrfD8Vwpig+h66uv?k znG0vcSBXjXnfN1dm09AKWTsW_zRR@E&_pr9E;NI@+x;rRZDz(s8`%dPO_YM zH+7^hg(N_dG6yseIVQ_1)|SiRBxXF##>8mKWEpWeHLn%P(V;+gM0}dMb(U_*>((YN zU&+bK+_yXIOq1g^f;I9mZZ<5>$5vpSbi}O5U?w2vt)6byCH8wv=C;1>DKH#^na=fJ zB_Fx71cxc;Iw_>iC~jcnVBayUom1^0qz2KIGL|})z=)~q) z5v*4y*6nGa>6)x9Z#dg_WI;3IWG}d z1Z4sjAs^ 0: print( - "Warning [vfc_ci]: Some of your runs could not generate any data " \ + "Warning [vfc_ci]: Some of your runs could not generate any data " "(for instance because your code crashed) and resulted in " "warnings. Here is the complete list :" ) @@ -316,9 +301,7 @@ def show_warnings(warnings): print(" Repetition: %s" % warnings[i]["repetition"]) - -################################################################################ - +########################################################################## # Entry point @@ -334,54 +317,51 @@ def run(is_git_commit, export_raw_values, dry_run): data, warnings = run_tests(config) show_warnings(warnings) - # Data processing print("Info [vfc_ci]: Processing data...") data_processing(data) - - # Prepare data for export (by creating a proper index and linking run timestamp) + # Prepare data for export (by creating a proper index and linking run + # timestamp) data = data.set_index(["test", "variable", "vfc_backend"]).sort_index() data["timestamp"] = metadata["timestamp"] - filename = metadata["hash"] if is_git_commit else str(metadata["timestamp"]) - + filename = metadata["hash"] if is_git_commit else str( + metadata["timestamp"]) # Prepare metadata for export metadata = pd.DataFrame.from_dict([metadata]) metadata = metadata.set_index("timestamp") - # NOTE : Exporting to HDF5 requires to install "tables" on the system # Export raw data if needed if export_raw_values and not dry_run: - data.to_hdf(filename + ".vfcraw.hd5", key="data") - metadata.to_hdf(filename + ".vfcraw.hd5", key="metadata") + data.to_hdf(filename + ".vfcraw.h5", key="data") + metadata.to_hdf(filename + ".vfcraw.h5", key="metadata") # Export data del data["values"] if not dry_run: - data.to_hdf(filename + ".vfcrun.hd5", key="data") - metadata.to_hdf(filename + ".vfcrun.hd5", key="metadata") - + data.to_hdf(filename + ".vfcrun.h5", key="data") + metadata.to_hdf(filename + ".vfcrun.h5", key="metadata") # Print termination messages print( - "Info [vfc_ci]: The results have been successfully written to " \ - "%s.vfcrun.hd5." \ - % filename - ) + "Info [vfc_ci]: The results have been successfully written to " + "%s.vfcrun.h5." + % filename + ) if export_raw_values: print( - "Info [vfc_ci]: A file containing the raw values has also been " \ - "created : %s.vfcraw.hd5." + "Info [vfc_ci]: A file containing the raw values has also been " + "created : %s.vfcraw.h5." % filename ) if dry_run: print( - "Info [vfc_ci]: The dry run flag was enabled, so no files were " \ + "Info [vfc_ci]: The dry run flag was enabled, so no files were " "actually created." ) diff --git a/ci/vfc_ci_report/__pycache__/compare_runs.cpython-39.pyc b/ci/vfc_ci_report/__pycache__/compare_runs.cpython-39.pyc index 0be4ed463e7dfb486022766db9121075b31904b2..faade86f027048e2829586b4a93c46dbfb88db59 100644 GIT binary patch delta 684 zcmZ`$&ubGw6yEpd?PQp2HX%)7s%D$CUSbP^2wr*+Y9Ss(JSvEW#7J7`A&H(MLJNZ6 z5AcCM@TSFcWxaUtAmTv~Jo^t=K@Xlvpt|eKZdE)u55Dr_=lxCD8I`okI203NA#bWC;Hg_E%W-w8H*8Y;NZ{*$6^Dzuu6kEx3V=j z$-V4pc)%~SXSD7Q`)k8dd#%G6Z?o~-c5g#*GFo_`k+6Usknl~vKKg<`c*l7mtAfP4 zI2K))cCL%D7&|N2#c5|ntct-=p3k-65#PzZEx)&x)zs1in%ePYe+b+B$$t&s!iV`2 zDnGHZ`&NeJlHkw#njpAPJz#L5S_8BgUa1D|%sLq9x|O=hwk1jl1{R4lZ`NiP_1<5a zv?3DtgCfcl2~El>GATnT;aBx&f6@P1RO&Dp0QBuDao5RZkU^NXq4_b00ze0t~o*ApigX delta 613 zcmY+AL1+^}6o&WByq%1j*-hMT!WO$pnucOZi#=4l3WDZRg?bcfBdKVDP2Gfo5b@Ac zun6{D#AEf~O=Nr3gQ6Fu9zE*Clf8)`78_@_h&Vqy-kW*<_lEbg@wVYLUDvVnnXBDj z*=xM;atJF_!kNP58>{iH=Gxq1H;!-ix@%XjcjFFS#8amu-QJruCXT@V=|DIF0+8He zAp~4Pjpuc3ROHEPtaDgS_A@(d>VhF$GXxZ%AQ-U`+7f)*(QbzeC?Y*~3a+LU!PEW; z?%VX)iH;Q%mcWRj>;lcYGT4MZbXYvyX9K4!RVa#vsUEdxkl!GO15+`7vLd|k4PX{o+&`2Ew2Y(w{tJyTk3vk4_YC1sqHm|%LxnyBX|9g) z|0|p7r4f9mdie=MMtkK4EW>EIvc()m`;{TM$;RY47#m}MH^l@KpoR3In$(N>Grfuq eG9zc?6; z*^hBOGb797e=O$QEL@B{EIdp+i~=l^9atL~^EV%2)n{ZZ-TaA7l##K0G6zQ=lN2KyG8NfPF5pULw3~c|%bM4njggI+gN;Loho6H5 E0Af8P`~Uy| diff --git a/ci/vfc_ci_report/__pycache__/inspect_runs.cpython-39.pyc b/ci/vfc_ci_report/__pycache__/inspect_runs.cpython-39.pyc index 4c7717c0c780adbd0504d5ea6f7de8a68e003505..1f6f2f40b72ca83d6743ada64fa8be32d39735b3 100644 GIT binary patch delta 645 zcmaJ;F>4e-6wdeN?d-dCcR4rb-E!p4kn=)3A=oGuf&?oI4H3JDMgoB#7|B79R2o52 zC<@aALC``Dv5JF*MYIsHva#_81nXcCRN~z^a}Y3%^Wb~TH}k&ty_rYTZ>N(RNfHgY zRqsBYTiZ;Q2{Yy{we-ru{Pp=o^MOvGYIo@)(6aBi2AsCvf?eRE{Stn{)OlChKH~)x zgZo4px*LUMib$aQ5(pKkL^=E}9dP@Y!?9$y3ZK=Xdrn|^(!pxW{K$vRP8OI>GK)3S zNoR21qq&k^#=8BTR+S1DmP{oZo4x5O+Rq&fQG}?^D%#J?u44@9ai0U)8FN`VvUi+Pz%7Pt%_tGTylcd|`j+rB7m;jn`9Q1>2G ziWD)~rSc_U%)GBW!&!4RU&oaFo|l3X7yQkQfm>$L&KXBwkoSEsYjd(*Ij0Cd1pFt~ zejXX@X;16yKWDyzdYbKA{WaG*zUc$sG>n3QU{?yM`xK#^%tWJhJoaqbi#c;P2;Ae) z4MLvf1FX5#s39;98yl$D=CP+39w8?_I+)14tbbo+zBb=9k2tD5;=uiXQDRACLhi_@ S9F?kLM6Stg3CyeUZRo!qA9aKP delta 556 zcmXX?ziSjh81>D3JNwWjV8(Bj-i@5y3(bEJd&|Mq?>rQbhcjgFRFbA}J&) zYBd%?uoeW3-e{?fiWX`77Ze0*vCt%UnK?6$nfd0MkM|z$-Sp$>XfBF^9d>n^4`=f? zQ7JI1m&Cr|vzO+}>vIc}Q?0VR+-@ygxY#PM>eu2hcA1~zBXGdHpd2`2zIuOvljei} z8T*f0ruMkwlXa@fKrmE~6^j9piVUNDWXQnMe5!nr`dnPmPof*Rt6#-u^vie#%dWzM z&d!=7Zrz;Mc`|M3BaUK@FaWBN*VA1BQ3 zEcJ>RH+iS;i9A+TpR6b6%WZwA$TAXeV9-XW2#I@bDlgIZirhRZo>RW+NR}OmA9(%g zO%7rk7mG~l(~W#6a^1KaQX&IeQI}XRHHOD{I{+K8=!H^98-!I!5ZK5`>9>s)Oqi+N zPcbwptVq4@)CSdf-J)hY0iWkv?CWn6Kj3cx9SCa< diff --git a/ci/vfc_ci_report/__pycache__/plot.cpython-39.pyc b/ci/vfc_ci_report/__pycache__/plot.cpython-39.pyc index 9e4b5971eaf6cb3b45368078cade265fce44ad32..db14f8d270b175d19cc3e2b3d74073db82349149 100644 GIT binary patch delta 235 zcmbO(JVBT*k(ZZ?0SM&&_9rk2PUPFqRLVH%EDVurXc`>PM<|4HkmKx4@#u|nNOf{Sf85tRB zSW_5)vj59~YBrZME@fhjm@LbpB^sQOT9liZr|X+vRFqkqpBJB&Uz9!Bm&IC{g@aLm zk&A_ek%f_ik%y7xFB=C7BNHRfKQ?w2MiG!K%jDH8#z6BXKVX4d$_}-Z9bxHY0oHpU zTR*T)W|DRXy0 len(timestamps): n = len(timestamps) - for i in range(0, n): # Get metadata associated to this run - row_metadata = helper.get_metadata(self.metadata, timestamps[-i-1]) - date = time.ctime(timestamps[-i-1]) + row_metadata = helper.get_metadata( + self.metadata, timestamps[-i - 1]) + date = time.ctime(timestamps[-i - 1]) # Fill the x series str = row_metadata["name"] - x_series.insert(0, helper.get_metadata(self.metadata, timestamps[-i-1])["name"]) + x_series.insert(0, helper.get_metadata( + self.metadata, timestamps[-i - 1])["name"]) # Fill the metadata lists x_metadata["date"].insert(0, date) - x_metadata["is_git_commit"].insert(0, row_metadata["is_git_commit"]) + x_metadata["is_git_commit"].insert( + 0, row_metadata["is_git_commit"]) x_metadata["hash"].insert(0, row_metadata["hash"]) x_metadata["author"].insert(0, row_metadata["author"]) x_metadata["message"].insert(0, row_metadata["message"]) - return x_series, x_metadata - - # Plots update function def update_plots(self): - # Select all data matching current test/var/backend + # Select all data matching current test/var/backend - runs = self.data.loc[ - [self.widgets["select_test"].value], - self.widgets["select_var"].value, self.widgets["select_backend"].value - ] + runs = self.data.loc[[self.widgets["select_test"].value], + self.widgets["select_var"].value, + self.widgets["select_backend"].value] timestamps = runs["timestamp"] x_series, x_metadata = self.gen_x_series(timestamps.sort_values()) - - # Update source + # Update source main_dict = runs.to_dict("series") main_dict["x"] = x_series @@ -91,8 +87,7 @@ class CompareRuns: # Select the last n runs only n = self.current_n_runs - main_dict = {key:value[-n:] for key, value in main_dict.items()} - + main_dict = {key: value[-n:] for key, value in main_dict.items()} # Generate ColumnDataSources for the 3 dotplots for stat in ["sigma", "s10", "s2"]: @@ -111,18 +106,20 @@ class CompareRuns: } if stat == "s10" or stat == "s2": - dict["%s_lower_bound" % stat] = main_dict["%s_lower_bound" % stat] + dict["%s_lower_bound" % + stat] = main_dict["%s_lower_bound" % + stat] # Filter outliers if the box is checked if len(self.widgets["outliers_filtering_compare"].active) > 0: outliers = helper.detect_outliers(dict[stat]) dict[stat] = helper.remove_outliers(dict[stat], outliers) - dict["%s_x" % stat] = helper.remove_outliers(dict["%s_x" % stat], outliers) + dict["%s_x" % stat] = helper.remove_outliers( + dict["%s_x" % stat], outliers) # Assign ColumnDataSource self.sources["%s_source" % stat].data = dict - # Generate ColumnDataSource for the boxplot dict = { "is_git_commit": main_dict["is_git_commit"], @@ -132,40 +129,48 @@ class CompareRuns: "message": main_dict["message"], "x": main_dict["x"], - "min" : main_dict["min"], - "quantile25" : main_dict["quantile25"], - "quantile50" : main_dict["quantile50"], - "quantile75" : main_dict["quantile75"], - "max" : main_dict["max"], - "mu" : main_dict["mu"], - "pvalue" : main_dict["pvalue"], + "min": main_dict["min"], + "quantile25": main_dict["quantile25"], + "quantile50": main_dict["quantile50"], + "quantile75": main_dict["quantile75"], + "max": main_dict["max"], + "mu": main_dict["mu"], + "pvalue": main_dict["pvalue"], "nsamples": main_dict["nsamples"] } - - self.sources["boxplot_source"].data = dict + # Update x axis - # Update x_ranges - helper.reset_x_range(self.plots["boxplot"], self.sources["boxplot_source"].data["x"]) - helper.reset_x_range(self.plots["sigma_plot"], self.sources["sigma_source"].data["sigma_x"]) - helper.reset_x_range(self.plots["s10_plot"], self.sources["s10_source"].data["s10_x"]) - helper.reset_x_range(self.plots["s2_plot"], self.sources["s2_source"].data["s2_x"]) - - - + helper.reset_x_range( + self.plots["boxplot"], + self.sources["boxplot_source"].data["x"] + ) + helper.reset_x_range( + self.plots["sigma_plot"], + self.sources["sigma_source"].data["sigma_x"] + ) + helper.reset_x_range( + self.plots["s10_plot"], + self.sources["s10_source"].data["s10_x"] + ) + helper.reset_x_range( + self.plots["s2_plot"], + self.sources["s2_source"].data["s2_x"] + ) # Widgets' callback functions def update_test(self, attrname, old, new): # If the value is updated by the CustomJS, self.widgets["select_var"].value - # won't be updated, so we have to look for that case and assign it manually + # won't be updated, so we have to look for that case and assign it + # manually # "new" should be a list when updated by CustomJS - if type(new) == list: + if isinstance(new, list): # If filtering removed all options, we might have an empty list # (in this case, we just skip the callback and do nothing) if len(new) > 0: @@ -180,10 +185,9 @@ class CompareRuns: # New list of available vars self.vars = self.data.loc[new]\ - .index.get_level_values("variable").drop_duplicates().tolist() + .index.get_level_values("variable").drop_duplicates().tolist() self.widgets["select_var"].options = self.vars - # Reset var selection if old one is not available in new vars if self.widgets["select_var"].value not in self.vars: self.widgets["select_var"].value = self.vars[0] @@ -194,14 +198,14 @@ class CompareRuns: # anyway) self.update_var("", "", self.widgets["select_var"].value) - def update_var(self, attrname, old, new): # If the value is updated by the CustomJS, self.widgets["select_var"].value - # won't be updated, so we have to look for that case and assign it manually + # won't be updated, so we have to look for that case and assign it + # manually # new should be a list when updated by CustomJS - if type(new) == list: + if isinstance(new, list): new = new[0] if new != self.widgets["select_var"].value: @@ -209,10 +213,9 @@ class CompareRuns: self.widgets["select_var"].value = new return - # New list of available backends self.backends = self.data.loc[self.widgets["select_test"].value, self.widgets["select_var"].value]\ - .index.get_level_values("vfc_backend").drop_duplicates().tolist() + .index.get_level_values("vfc_backend").drop_duplicates().tolist() self.widgets["select_backend"].options = self.backends # Reset backend selection if old one is not available in new backends @@ -225,13 +228,11 @@ class CompareRuns: # anyway) self.update_backend("", "", self.widgets["select_backend"].value) - def update_backend(self, attrname, old, new): # Simply update plots, since no other data is affected self.update_plots() - def update_n_runs(self, attrname, old, new): # Simply update runs selection (value and string display) self.select_n_runs.value = new @@ -239,12 +240,9 @@ class CompareRuns: self.update_plots() - def update_outliers_filtering(self, attrname, old, new): self.update_plots() - - # Bokeh setup functions def setup_plots(self): @@ -256,7 +254,6 @@ class CompareRuns: # (defined inside template to avoid bloating server w/ too much JS code) js_tap_callback = "goToInspectRuns();" - # Box plot self.plots["boxplot"] = figure( name="boxplot", title="Variable distribution over runs", @@ -280,24 +277,23 @@ class CompareRuns: ("Number of samples", "@nsamples") ] box_tooltips_formatters = { - "@min" : "printf", - "@max" : "printf", - "@quantile25" : "printf", - "@quantile50" : "printf", - "@quantile75" : "printf", - "@mu" : "printf" + "@min": "printf", + "@max": "printf", + "@quantile25": "printf", + "@quantile50": "printf", + "@quantile75": "printf", + "@mu": "printf" } plot.fill_boxplot( self.plots["boxplot"], self.sources["boxplot_source"], - tooltips = box_tooltips, - tooltips_formatters = box_tooltips_formatters, - js_tap_callback = js_tap_callback, - server_tap_callback = self.inspect_run_callback_boxplot, + tooltips=box_tooltips, + tooltips_formatters=box_tooltips_formatters, + js_tap_callback=js_tap_callback, + server_tap_callback=self.inspect_run_callback_boxplot, ) self.doc.add_root(self.plots["boxplot"]) - # Sigma plot (bar plot) self.plots["sigma_plot"] = figure( name="sigma_plot", title="Standard deviation σ over runs", @@ -317,14 +313,13 @@ class CompareRuns: plot.fill_dotplot( self.plots["sigma_plot"], self.sources["sigma_source"], "sigma", - tooltips = sigma_tooltips, - js_tap_callback = js_tap_callback, - server_tap_callback = self.inspect_run_callback_sigma, - lines = True + tooltips=sigma_tooltips, + js_tap_callback=js_tap_callback, + server_tap_callback=self.inspect_run_callback_sigma, + lines=True ) self.doc.add_root(self.plots["sigma_plot"]) - # s plot (bar plot with 2 tabs) self.plots["s10_plot"] = figure( name="s10_plot", title="Significant digits s over runs", @@ -345,15 +340,14 @@ class CompareRuns: plot.fill_dotplot( self.plots["s10_plot"], self.sources["s10_source"], "s10", - tooltips = s10_tooltips, - js_tap_callback = js_tap_callback, - server_tap_callback = self.inspect_run_callback_s10, - lines = True, + tooltips=s10_tooltips, + js_tap_callback=js_tap_callback, + server_tap_callback=self.inspect_run_callback_s10, + lines=True, lower_bound=True ) s10_tab = Panel(child=self.plots["s10_plot"], title="Base 10") - self.plots["s2_plot"] = figure( name="s2_plot", title="Significant digits s over runs", plot_width=900, plot_height=400, x_range=[""], @@ -373,45 +367,42 @@ class CompareRuns: plot.fill_dotplot( self.plots["s2_plot"], self.sources["s2_source"], "s2", - tooltips = s2_tooltips, - js_tap_callback = js_tap_callback, - server_tap_callback = self.inspect_run_callback_s2, - lines = True, + tooltips=s2_tooltips, + js_tap_callback=js_tap_callback, + server_tap_callback=self.inspect_run_callback_s2, + lines=True, lower_bound=True ) s2_tab = Panel(child=self.plots["s2_plot"], title="Base 2") s_tabs = Tabs( - name = "s_tabs", + name="s_tabs", tabs=[s10_tab, s2_tab], - tabs_location = "below" + tabs_location="below" ) self.doc.add_root(s_tabs) - def setup_widgets(self): - # Initial selections + # Initial selections # Test/var/backend combination (we select all first elements at init) self.tests = self.data\ - .index.get_level_values("test").drop_duplicates().tolist() + .index.get_level_values("test").drop_duplicates().tolist() self.vars = self.data.loc[self.tests[0]]\ - .index.get_level_values("variable").drop_duplicates().tolist() + .index.get_level_values("variable").drop_duplicates().tolist() self.backends = self.data.loc[self.tests[0], self.vars[0]]\ - .index.get_level_values("vfc_backend").drop_duplicates().tolist() - + .index.get_level_values("vfc_backend").drop_duplicates().tolist() # Custom JS callback that will be used client side to filter selections filter_callback_js = """ selector.options = options.filter(e => e.includes(cb_obj.value)); """ - - # Test selector widget + # Test selector widget # Number of runs to display # The dict structure allows us to get int value from the display string @@ -442,14 +433,16 @@ class CompareRuns: self.widgets["test_filter"] = TextInput( name="test_filter", title="Tests filter:" ) - self.widgets["test_filter"].js_on_change("value", CustomJS( - args=dict(options=self.tests, selector=self.widgets["select_test"]), - code=filter_callback_js - )) + self.widgets["test_filter"].js_on_change( + "value", + CustomJS( + args=dict( + options=self.tests, + selector=self.widgets["select_test"]), + code=filter_callback_js)) self.doc.add_root(self.widgets["test_filter"]) - - # Number of runs to display + # Number of runs to display self.widgets["select_n_runs"] = Select( name="select_n_runs", title="Display :", @@ -458,8 +451,7 @@ class CompareRuns: self.doc.add_root(self.widgets["select_n_runs"]) self.widgets["select_n_runs"].on_change("value", self.update_n_runs) - - # Variable selector widget + # Variable selector widget self.widgets["select_var"] = Select( name="select_var", title="Variable :", @@ -469,8 +461,7 @@ class CompareRuns: self.widgets["select_var"].on_change("value", self.update_var) self.widgets["select_var"].on_change("options", self.update_var) - - # Backend selector widget + # Backend selector widget self.widgets["select_backend"] = Select( name="select_backend", title="Verificarlo backend :", @@ -479,23 +470,21 @@ class CompareRuns: self.doc.add_root(self.widgets["select_backend"]) self.widgets["select_backend"].on_change("value", self.update_backend) - - # Outliers filtering checkbox + # Outliers filtering checkbox self.widgets["outliers_filtering_compare"] = CheckboxGroup( name="outliers_filtering_compare", - labels=["Filter outliers"], active =[] + labels=["Filter outliers"], active=[] ) self.doc.add_root(self.widgets["outliers_filtering_compare"]) self.widgets["outliers_filtering_compare"]\ - .on_change("active", self.update_outliers_filtering) - - + .on_change("active", self.update_outliers_filtering) # Communication methods # (to send/receive messages to/from master) # Callback to change view of Inspect runs when data is selected + def inspect_run_callback(self, new, source_name, x_name): # In case we just unselected everything, then do nothing @@ -507,7 +496,6 @@ class CompareRuns: self.master.go_to_inspect(run_name) - # Wrappers for each plot (since new is the index of the clicked element, # it is dependent of the plot because we could have filtered some outliers) # There doesn't seem to be an easy way to add custom parameters to a @@ -525,7 +513,6 @@ class CompareRuns: def inspect_run_callback_s10(self, attr, old, new): self.inspect_run_callback(new, "s10_source", "s10_x") - # Constructor def __init__(self, master, doc, data, metadata): @@ -536,11 +523,10 @@ class CompareRuns: self.data = data self.metadata = metadata - self.sources = { "boxplot_source": ColumnDataSource(data={}), "sigma_source": ColumnDataSource(data={}), - "s10_source" :ColumnDataSource(data={}), + "s10_source": ColumnDataSource(data={}), "s2_source": ColumnDataSource(data={}) } diff --git a/ci/vfc_ci_report/helper.py b/ci/vfc_ci_report/helper.py index 57c84d7..f258959 100644 --- a/ci/vfc_ci_report/helper.py +++ b/ci/vfc_ci_report/helper.py @@ -10,7 +10,7 @@ import numpy as np max_ticks = 15 max_zscore = 3 -################################################################################ +########################################################################## # From a timestamp, return the associated metadata as a Pandas serie @@ -39,7 +39,6 @@ def get_run_name(timestamp, hash): now = calendar.timegm(gmt) diff = now - timestamp - # Special case : < 1 minute (return string directly) if diff < 60: str = "Less than a minute ago" @@ -83,12 +82,10 @@ def get_run_name(timestamp, hash): str = str % (n, plural) - # We might want to add the git hash if hash != "": str = str + " (%s)" % hash - # Finally, check for duplicate with previously generated string if str == get_run_name.previous: # Increment the duplicate counter and add it to str @@ -96,12 +93,14 @@ def get_run_name(timestamp, hash): str = "%s (%s)" % (str, get_run_name.counter) else: - # No duplicate, reset both previously generated str and duplicate counter + # No duplicate, reset both previously generated str and duplicate + # counter get_run_name.counter = 0 get_run_name.previous = str return str + # These external variables will store data about the last generated string to # avoid duplicates (assuming the runs are sorted by time) get_run_name.counter = 0 @@ -156,11 +155,16 @@ def remove_boxplot_outliers(dict, outliers, prefix): dict["%s_x" % prefix] = remove_outliers(dict["%s_x" % prefix], outliers) - dict["%s_min" % prefix] = remove_outliers(dict["%s_min" % prefix], outliers) - dict["%s_quantile25" % prefix] = remove_outliers(dict["%s_quantile25" % prefix], outliers) - dict["%s_quantile50" % prefix] = remove_outliers(dict["%s_quantile50" % prefix], outliers) - dict["%s_quantile75" % prefix] = remove_outliers(dict["%s_quantile75" % prefix], outliers) - dict["%s_max" % prefix] = remove_outliers(dict["%s_max" % prefix], outliers) + dict["%s_min" % prefix] = remove_outliers( + dict["%s_min" % prefix], outliers) + dict["%s_quantile25" % prefix] = remove_outliers( + dict["%s_quantile25" % prefix], outliers) + dict["%s_quantile50" % prefix] = remove_outliers( + dict["%s_quantile50" % prefix], outliers) + dict["%s_quantile75" % prefix] = remove_outliers( + dict["%s_quantile75" % prefix], outliers) + dict["%s_max" % prefix] = remove_outliers( + dict["%s_max" % prefix], outliers) dict["%s_mu" % prefix] = remove_outliers(dict["%s_mu" % prefix], outliers) dict["nsamples"] = remove_outliers(dict["nsamples"], outliers) diff --git a/ci/vfc_ci_report/inspect_runs.py b/ci/vfc_ci_report/inspect_runs.py index 57a1caa..f53ab17 100644 --- a/ci/vfc_ci_report/inspect_runs.py +++ b/ci/vfc_ci_report/inspect_runs.py @@ -9,19 +9,18 @@ import numpy as np from bokeh.plotting import figure, curdoc from bokeh.embed import components from bokeh.models import Select, ColumnDataSource, Panel, Tabs, HoverTool,\ -RadioButtonGroup, CheckboxGroup, CustomJS + RadioButtonGroup, CheckboxGroup, CustomJS import helper import plot -################################################################################ - +########################################################################## class InspectRuns: - # Helper functions related to InspectRun + # Helper functions related to InspectRun # Returns a dictionary mapping user-readable strings to all run timestamps def gen_runs_selection(self): @@ -40,7 +39,6 @@ class InspectRuns: return runs_dict - def gen_boxplot_tooltips(self, prefix): return [ ("Name", "@%s_x" % prefix), @@ -55,34 +53,35 @@ class InspectRuns: def gen_boxplot_tooltips_formatters(self, prefix): return { - "@%s_min" % prefix : "printf", - "@%s_max" % prefix : "printf", - "@%s_quantile25" % prefix : "printf", - "@%s_quantile50" % prefix : "printf", - "@%s_quantile75" % prefix : "printf", - "@%s_mu" % prefix : "printf" + "@%s_min" % prefix: "printf", + "@%s_max" % prefix: "printf", + "@%s_quantile25" % prefix: "printf", + "@%s_quantile50" % prefix: "printf", + "@%s_quantile75" % prefix: "printf", + "@%s_mu" % prefix: "printf" } - # Data processing helper # (computes new distributions for sigma, s2, s10) + def data_processing(self, dataframe): # Compute aggragated mu - dataframe["mu"] = np.vectorize(np.average)(dataframe["mu"], weights=dataframe["nsamples"]) + dataframe["mu"] = np.vectorize( + np.average)( + dataframe["mu"], + weights=dataframe["nsamples"]) # nsamples is the number of aggregated elements (as well as the number # of samples for our new sigma and s distributions) dataframe["nsamples"] = dataframe["nsamples"].apply(lambda x: len(x)) - dataframe["mu_x"] = dataframe.index # Make sure that strings don't excede a certain length dataframe["mu_x"] = dataframe["mu_x"].apply( lambda x: x[:17] + "[...]" + x[-17:] if len(x) > 39 else x ) - # Get quantiles and mu for sigma, s10, s2 for prefix in ["sigma", "s10", "s2"]: @@ -91,19 +90,19 @@ class InspectRuns: dataframe[prefix] = dataframe[prefix].apply(np.sort) dataframe["%s_min" % prefix] = dataframe[prefix].apply(np.min) - dataframe["%s_quantile25" % prefix] = dataframe[prefix].apply(np.quantile, args=(0.25,)) - dataframe["%s_quantile50" % prefix] = dataframe[prefix].apply(np.quantile, args=(0.50,)) - dataframe["%s_quantile75" % prefix] = dataframe[prefix].apply(np.quantile, args=(0.75,)) + dataframe["%s_quantile25" % prefix] = dataframe[prefix].apply( + np.quantile, args=(0.25,)) + dataframe["%s_quantile50" % prefix] = dataframe[prefix].apply( + np.quantile, args=(0.50,)) + dataframe["%s_quantile75" % prefix] = dataframe[prefix].apply( + np.quantile, args=(0.75,)) dataframe["%s_max" % prefix] = dataframe[prefix].apply(np.max) dataframe["%s_mu" % prefix] = dataframe[prefix].apply(np.average) del dataframe[prefix] - return dataframe - - - # Plots update function + # Plots update function def update_plots(self): @@ -117,8 +116,7 @@ class InspectRuns: ] filterby = self.factors_dict[filterby_display] - - # Groupby and aggregate lines belonging to the same group in lists + # Groupby and aggregate lines belonging to the same group in lists groups = self.run_data[ self.run_data.index.isin( @@ -131,32 +129,31 @@ class InspectRuns: "sigma": lambda x: x.tolist(), "s10": lambda x: x.tolist(), "s2": lambda x: x.tolist(), + "mu": lambda x: x.tolist(), # Used for mu weighted average first, then will be replaced "nsamples": lambda x: x.tolist() }) - - # Compute the new distributions, ... + # Compute the new distributions, ... groups = self.data_processing(groups).to_dict("list") - - # Update source + # Update source # Assign each ColumnDataSource, starting with the boxplots for prefix in ["sigma", "s10", "s2"]: dict = { - "%s_x" % prefix: groups["%s_x" % prefix], - "%s_min" % prefix: groups["%s_min" % prefix], - "%s_quantile25" % prefix: groups["%s_quantile25" % prefix], - "%s_quantile50" % prefix: groups["%s_quantile50" % prefix], - "%s_quantile75" % prefix: groups["%s_quantile75" % prefix], - "%s_max" % prefix: groups["%s_max" % prefix], - "%s_mu" % prefix: groups["%s_mu" % prefix], + "%s_x" % prefix: groups["%s_x" % prefix], + "%s_min" % prefix: groups["%s_min" % prefix], + "%s_quantile25" % prefix: groups["%s_quantile25" % prefix], + "%s_quantile50" % prefix: groups["%s_quantile50" % prefix], + "%s_quantile75" % prefix: groups["%s_quantile75" % prefix], + "%s_max" % prefix: groups["%s_max" % prefix], + "%s_mu" % prefix: groups["%s_mu" % prefix], - "nsamples": groups["nsamples"] + "nsamples": groups["nsamples"] } # Filter outliers if the box is checked @@ -166,7 +163,8 @@ class InspectRuns: top_outliers = helper.detect_outliers(dict["%s_max" % prefix]) helper.remove_boxplot_outliers(dict, top_outliers, prefix) - bottom_outliers = helper.detect_outliers(dict["%s_min" % prefix]) + bottom_outliers = helper.detect_outliers( + dict["%s_min" % prefix]) helper.remove_boxplot_outliers(dict, bottom_outliers, prefix) self.sources["%s_source" % prefix].data = dict @@ -185,8 +183,8 @@ class InspectRuns: if len(self.widgets["outliers_filtering_inspect"].active) > 0: mu_outliers = helper.detect_outliers(groups["mu"]) groups["mu"] = helper.remove_outliers(groups["mu"], mu_outliers) - groups["mu_x"] = helper.remove_outliers(groups["mu_x"], mu_outliers) - + groups["mu_x"] = helper.remove_outliers( + groups["mu_x"], mu_outliers) # Update plots axis/titles @@ -194,42 +192,38 @@ class InspectRuns: factors_dict = self.factors_dict.copy() del factors_dict[groupby_display] del factors_dict[filterby_display] - over_all = list(factors_dict.keys())[0] + for_all = list(factors_dict.keys())[0] # Update all display strings for plot title (remove caps, plural) groupby_display = groupby_display.lower() filterby_display = filterby_display.lower()[:-1] - over_all = over_all.lower() + for_all = for_all.lower() self.plots["mu_inspect"].title.text = \ - "Empirical average μ of %s (groupped by %s, for all %s)" \ - % (filterby_display, groupby_display, over_all) + "Empirical average μ of %s (groupped by %s, for all %s)" \ + % (filterby_display, groupby_display, for_all) self.plots["sigma_inspect"].title.text = \ - "Standard deviation σ of %s (groupped by %s, for all %s)" \ - % (filterby_display, groupby_display, over_all) + "Standard deviation σ of %s (groupped by %s, for all %s)" \ + % (filterby_display, groupby_display, for_all) self.plots["s10_inspect"].title.text = \ - "Significant digits s of %s (groupped by %s, for all %s)" \ - % (filterby_display, groupby_display, over_all) + "Significant digits s of %s (groupped by %s, for all %s)" \ + % (filterby_display, groupby_display, for_all) self.plots["s2_inspect"].title.text = \ - "Significant digits s of %s (groupped by %s, for all %s)" \ - % (filterby_display, groupby_display, over_all) - - - # Update x_ranges + "Significant digits s of %s (groupped by %s, for all %s)" \ + % (filterby_display, groupby_display, for_all) helper.reset_x_range(self.plots["mu_inspect"], groups["mu_x"]) helper.reset_x_range(self.plots["sigma_inspect"], groups["sigma_x"]) helper.reset_x_range(self.plots["s10_inspect"], groups["s10_x"]) helper.reset_x_range(self.plots["s2_inspect"], groups["s2_x"]) - - # Widets' callback functions # Run selector callback + def update_run(self, attrname, old, new): filterby = self.widgets["filterby_radio"].labels[ @@ -248,7 +242,7 @@ class InspectRuns: # Update filter options options = self.run_data.index\ - .get_level_values(filterby).drop_duplicates().tolist() + .get_level_values(filterby).drop_duplicates().tolist() self.widgets["select_filter"].options = options if old_value not in self.widgets["select_filter"].options: @@ -260,8 +254,8 @@ class InspectRuns: # anyway) self.update_filter("", "", old_value) - # "Group by" radio + def update_groupby(self, attrname, old, new): # Update "Filter by" radio list @@ -269,7 +263,6 @@ class InspectRuns: del filterby_list[self.widgets["groupby_radio"].active] self.widgets["filterby_radio"].labels = filterby_list - filterby = self.widgets["filterby_radio"].labels[ self.widgets["filterby_radio"].active ] @@ -280,7 +273,7 @@ class InspectRuns: # Update filter options options = self.run_data.index\ - .get_level_values(filterby).drop_duplicates().tolist() + .get_level_values(filterby).drop_duplicates().tolist() self.widgets["select_filter"].options = options if old_value not in self.widgets["select_filter"].options: @@ -292,8 +285,8 @@ class InspectRuns: # anyway) self.update_filter("", "", old_value) - # "Filter by" radio + def update_filterby(self, attrname, old, new): filterby = self.widgets["filterby_radio"].labels[ @@ -306,7 +299,7 @@ class InspectRuns: # Update filter selector options options = self.run_data.index\ - .get_level_values(filterby).drop_duplicates().tolist() + .get_level_values(filterby).drop_duplicates().tolist() self.widgets["select_filter"].options = options if old_value not in self.widgets["select_filter"].options: @@ -318,20 +311,18 @@ class InspectRuns: # anyway) self.update_filter("", "", old_value) - # Filter selector callback + def update_filter(self, attrname, old, new): self.update_plots() - # Filter outliers checkbox callback + def update_outliers_filtering(self, attrname, old, new): # The status (checked/unchecked) of the checkbox is also verified inside # self.update_plots(), so calling this function is enough self.update_plots() - - # Bokeh setup functions # (for both variable and backend selection at once) @@ -339,8 +330,7 @@ class InspectRuns: tools = "pan, wheel_zoom, xwheel_zoom, ywheel_zoom, reset, save" - - # Tooltips and formatters + # Tooltips and formatters dotplot_tooltips = [ ("Name", "@mu_x"), @@ -348,20 +338,22 @@ class InspectRuns: ("Number of samples (tests)", "@nsamples") ] dotplot_formatters = { - "@mu" : "printf" + "@mu": "printf" } sigma_boxplot_tooltips = self.gen_boxplot_tooltips("sigma") - sigma_boxplot_tooltips_formatters = self.gen_boxplot_tooltips_formatters("sigma") + sigma_boxplot_tooltips_formatters = self.gen_boxplot_tooltips_formatters( + "sigma") s10_boxplot_tooltips = self.gen_boxplot_tooltips("s10") - s10_boxplot_tooltips_formatters = self.gen_boxplot_tooltips_formatters("s10") + s10_boxplot_tooltips_formatters = self.gen_boxplot_tooltips_formatters( + "s10") s2_boxplot_tooltips = self.gen_boxplot_tooltips("s2") - s2_boxplot_tooltips_formatters = self.gen_boxplot_tooltips_formatters("s2") + s2_boxplot_tooltips_formatters = self.gen_boxplot_tooltips_formatters( + "s2") - - # Plots + # Plots # Mu plot self.plots["mu_inspect"] = figure( @@ -372,12 +364,11 @@ class InspectRuns: ) plot.fill_dotplot( self.plots["mu_inspect"], self.sources["mu_source"], "mu", - tooltips = dotplot_tooltips, - tooltips_formatters = dotplot_formatters + tooltips=dotplot_tooltips, + tooltips_formatters=dotplot_formatters ) self.doc.add_root(self.plots["mu_inspect"]) - # Sigma plot self.plots["sigma_inspect"] = figure( name="sigma_inspect", @@ -386,13 +377,13 @@ class InspectRuns: tools=tools, sizing_mode="scale_width" ) plot.fill_boxplot( - self.plots["sigma_inspect"], self.sources["sigma_source"], prefix="sigma", - tooltips = sigma_boxplot_tooltips, - tooltips_formatters = sigma_boxplot_tooltips_formatters - ) + self.plots["sigma_inspect"], + self.sources["sigma_source"], + prefix="sigma", + tooltips=sigma_boxplot_tooltips, + tooltips_formatters=sigma_boxplot_tooltips_formatters) self.doc.add_root(self.plots["sigma_inspect"]) - # s plots self.plots["s10_inspect"] = figure( name="s10_inspect", @@ -401,11 +392,14 @@ class InspectRuns: tools=tools, sizing_mode='scale_width' ) plot.fill_boxplot( - self.plots["s10_inspect"], self.sources["s10_source"], prefix="s10", - tooltips = s10_boxplot_tooltips, - tooltips_formatters = s10_boxplot_tooltips_formatters - ) - s10_tab_inspect = Panel(child=self.plots["s10_inspect"], title="Base 10") + self.plots["s10_inspect"], + self.sources["s10_source"], + prefix="s10", + tooltips=s10_boxplot_tooltips, + tooltips_formatters=s10_boxplot_tooltips_formatters) + s10_tab_inspect = Panel( + child=self.plots["s10_inspect"], + title="Base 10") self.plots["s2_inspect"] = figure( name="s2_inspect", @@ -415,22 +409,20 @@ class InspectRuns: ) plot.fill_boxplot( self.plots["s2_inspect"], self.sources["s2_source"], prefix="s2", - tooltips = s2_boxplot_tooltips, - tooltips_formatters = s2_boxplot_tooltips_formatters + tooltips=s2_boxplot_tooltips, + tooltips_formatters=s2_boxplot_tooltips_formatters ) s2_tab_inspect = Panel(child=self.plots["s2_inspect"], title="Base 2") s_tabs_inspect = Tabs( - name = "s_tabs_inspect", - tabs=[s10_tab_inspect, s2_tab_inspect], tabs_location = "below" + name="s_tabs_inspect", + tabs=[s10_tab_inspect, s2_tab_inspect], tabs_location="below" ) self.doc.add_root(s_tabs_inspect) - - def setup_widgets(self): - # Generation of selectable items + # Generation of selectable items # Dict contains all inspectable runs (maps display strings to timestamps) # The dict structure allows to get the timestamp from the display string @@ -445,8 +437,7 @@ class InspectRuns: "Tests": "test" } - - # Run selection + # Run selection # Contains all options strings runs_display = list(self.runs_dict.keys()) @@ -457,8 +448,7 @@ class InspectRuns: # This contains only entries matching the run self.run_data = self.data[self.data["timestamp"] == self.current_run] - - change_run_callback_js="updateRunMetadata(cb_obj.value);" + change_run_callback_js = "updateRunMetadata(cb_obj.value);" self.widgets["select_run"] = Select( name="select_run", title="Run :", @@ -467,7 +457,7 @@ class InspectRuns: self.doc.add_root(self.widgets["select_run"]) self.widgets["select_run"].on_change("value", self.update_run) self.widgets["select_run"].js_on_change("value", CustomJS( - code = change_run_callback_js, + code=change_run_callback_js, args=(dict( metadata=helper.metadata_to_dict( helper.get_metadata(self.metadata, self.current_run) @@ -475,8 +465,7 @@ class InspectRuns: )) )) - - # Factors selection + # Factors selection # "Group by" radio self.widgets["groupby_radio"] = RadioButtonGroup( @@ -491,7 +480,6 @@ class InspectRuns: self.update_groupby ) - # "Filter by" radio # Get all possible factors, and remove the one selected in "Group by" filterby_list = list(self.factors_dict.keys()) @@ -509,7 +497,6 @@ class InspectRuns: self.update_filterby ) - # Filter selector filterby = self.widgets["filterby_radio"].labels[ @@ -518,7 +505,7 @@ class InspectRuns: filterby = self.factors_dict[filterby] options = self.run_data.index\ - .get_level_values(filterby).drop_duplicates().tolist() + .get_level_values(filterby).drop_duplicates().tolist() self.widgets["select_filter"] = Select( # We need a different name to avoid collision in the template with @@ -528,30 +515,26 @@ class InspectRuns: ) self.doc.add_root(self.widgets["select_filter"]) self.widgets["select_filter"]\ - .on_change("value", self.update_filter) + .on_change("value", self.update_filter) - - # Toggle for outliers filtering + # Toggle for outliers filtering self.widgets["outliers_filtering_inspect"] = CheckboxGroup( name="outliers_filtering_inspect", - labels=["Filter outliers"], active = [] + labels=["Filter outliers"], active=[] ) self.doc.add_root(self.widgets["outliers_filtering_inspect"]) self.widgets["outliers_filtering_inspect"]\ - .on_change("active", self.update_outliers_filtering) - - + .on_change("active", self.update_outliers_filtering) # Communication methods # (to send/receive messages to/from master) # When received, switch to the run_name in parameter + def switch_view(self, run_name): self.widgets["select_run"].value = run_name - - # Constructor def __init__(self, master, doc, data, metadata): @@ -562,11 +545,10 @@ class InspectRuns: self.data = data self.metadata = metadata - self.sources = { "mu_source": ColumnDataSource(data={}), "sigma_source": ColumnDataSource(data={}), - "s10_source" :ColumnDataSource(data={}), + "s10_source": ColumnDataSource(data={}), "s2_source": ColumnDataSource(data={}) } diff --git a/ci/vfc_ci_report/main.py b/ci/vfc_ci_report/main.py index d011b3e..4af73d2 100644 --- a/ci/vfc_ci_report/main.py +++ b/ci/vfc_ci_report/main.py @@ -1,5 +1,5 @@ # Look for and read all the run files in the current directory (ending with -# .vfcrun.hd5), and lanch a Bokeh server for the visualization of this data. +# .vfcrunh5), and lanch a Bokeh server for the visualization of this data. import os import sys @@ -14,18 +14,16 @@ import compare_runs import inspect_runs import helper -################################################################################ +########################################################################## +# Read vfcrun files, and aggregate them in one dataset - # Read vfcrun files, and aggregate them in one dataset - -run_files = [ f for f in os.listdir(".") if f.endswith(".vfcrun.hd5") ] +run_files = [f for f in os.listdir(".") if f.endswith(".vfcrun.h5")] if len(run_files) == 0: print( - "Warning [vfc_ci]: Could not find any vfcrun files in the directory. " \ - "This will result in server errors and prevent you from viewing the report." - ) + "Warning [vfc_ci]: Could not find any vfcrun files in the directory. " + "This will result in server errors and prevent you from viewing the report.") # These are arrays of Pandas dataframes for now metadata = [] @@ -55,15 +53,14 @@ metadata["date"] = metadata.index.to_series().map( ) -################################################################################ +########################################################################## curdoc().title = "Verificarlo Report" - - # Read server arguments - # (this is quite easy because Bokeh server is called through a wrapper, so - # we know exactly what the arguments might be) +# Read server arguments +# (this is quite easy because Bokeh server is called through a wrapper, so +# we know exactly what the arguments might be) git_repo_linked = False commit_link = "" @@ -83,7 +80,6 @@ for i in range(1, len(sys.argv)): address = sys.argv[i + 2] url = "" - # Here, address is either the remote URL or the path to the local Git # repo (depending on the method) @@ -99,12 +95,11 @@ for i in range(1, len(sys.argv)): else: raise ValueError( - "Error [vfc_ci]: The specified method to get the Git " \ - "repository is invalid. Are you calling Bokeh directly " \ + "Error [vfc_ci]: The specified method to get the Git " + "repository is invalid. Are you calling Bokeh directly " "instead of using the Verificarlo wrapper ?" ) - # At this point, "url" should be set correctly, we can get the repo's # URL and name, after making sure we're on a Git URL @@ -113,7 +108,7 @@ for i in range(1, len(sys.argv)): path = parsed_url.path.split("/") if len(path) < 3: raise ValueError( - "Error [vfc_ci]: The found URL doesn't seem to be pointing " \ + "Error [vfc_ci]: The found URL doesn't seem to be pointing " "to a Git repository (path is too short)" ) @@ -122,12 +117,11 @@ for i in range(1, len(sys.argv)): curdoc().template_variables["repo_url"] = url curdoc().template_variables["repo_name"] = repo_name - # We should have a "github.com" or a "*gitlab*" URL if parsed_url.netloc == "github.com": commit_link = "https://%s%s/commit/" \ - % (parsed_url.netloc, parsed_url.path) + % (parsed_url.netloc, parsed_url.path) curdoc().template_variables["commit_link"] = commit_link curdoc().template_variables["git_host"] = "GitHub" @@ -138,7 +132,7 @@ for i in range(1, len(sys.argv)): # We assume we have a GitLab URL else: commit_link = "https://%s%s/-/commit/" \ - % (parsed_url.netloc, parsed_url.path) + % (parsed_url.netloc, parsed_url.path) curdoc().template_variables["commit_link"] = commit_link curdoc().template_variables["git_host"] = "GitLab" @@ -148,8 +142,6 @@ for i in range(1, len(sys.argv)): git_repo_linked = True - - # Look for a logo URL # If a logo URL is specified, it will be included in the report's header if sys.argv[i] == "logo": @@ -162,10 +154,9 @@ curdoc().template_variables["git_repo_linked"] = git_repo_linked curdoc().template_variables["has_logo"] = has_logo -################################################################################ +########################################################################## - - # Setup report views +# Setup report views # Define a ViewsMaster class to allow two-ways communication between views. # This approach by classes allows us to have separate scopes for each view and @@ -174,13 +165,12 @@ curdoc().template_variables["has_logo"] = has_logo class ViewsMaster: - # Communication functions + # Communication functions def go_to_inspect(self, run_name): self.inspect.switch_view(run_name) - - #Constructor + # Constructor def __init__(self, data, metadata, git_repo_linked, commit_link): @@ -190,28 +180,29 @@ class ViewsMaster: self.commit_link = commit_link # Pass metadata to the template as a JSON string - curdoc().template_variables["metadata"] = self.metadata.to_json(orient="index") + curdoc().template_variables["metadata"] = self.metadata.to_json( + orient="index") # Runs comparison self.compare = compare_runs.CompareRuns( - master = self, - doc = curdoc(), - data = data, - metadata = metadata, + master=self, + doc=curdoc(), + data=data, + metadata=metadata, ) # Runs inspection self.inspect = inspect_runs.InspectRuns( - master = self, - doc = curdoc(), - data = data, - metadata = metadata, + master=self, + doc=curdoc(), + data=data, + metadata=metadata, ) views_master = ViewsMaster( - data = data, - metadata = metadata, - git_repo_linked = git_repo_linked, - commit_link = commit_link + data=data, + metadata=metadata, + git_repo_linked=git_repo_linked, + commit_link=commit_link ) diff --git a/ci/vfc_ci_report/plot.py b/ci/vfc_ci_report/plot.py index 270266e..f4bb20f 100644 --- a/ci/vfc_ci_report/plot.py +++ b/ci/vfc_ci_report/plot.py @@ -15,21 +15,19 @@ def fill_dotplot( ): # (Optional) Tooltip and tooltip formatters - if tooltips != None: - hover = HoverTool(tooltips = tooltips, mode="vline", names=["circle"]) + if tooltips is not None: + hover = HoverTool(tooltips=tooltips, mode="vline", names=["circle"]) - if tooltips_formatters != None: + if tooltips_formatters is not None: hover.formatters = tooltips_formatters plot.add_tools(hover) - # (Optional) Add TapTool (for JS tap callback) - if js_tap_callback != None: + if js_tap_callback is not None: tap = TapTool(callback=CustomJS(code=js_tap_callback)) plot.add_tools(tap) - # (Optional) Add segment to represent a lower bound if lower_bound: lower_segment = plot.segment( @@ -38,24 +36,20 @@ def fill_dotplot( source=source, line_color="black" ) - # Draw dots (actually Bokeh circles) circle = plot.circle( name="circle", x="%s_x" % data_field, y=data_field, source=source, size=12 ) - # (Optional) Draw lines between dots if lines: line = plot.line(x="%s_x" % data_field, y=data_field, source=source) - # (Optional) Add server tap callback - if server_tap_callback != None: + if server_tap_callback is not None: circle.data_source.selected.on_change("indices", server_tap_callback) - # Plot appearance plot.xgrid.grid_line_color = None plot.ygrid.grid_line_color = None @@ -64,33 +58,30 @@ def fill_dotplot( plot.yaxis[0].formatter.power_limit_low = 0 plot.yaxis[0].formatter.precision = 3 - plot.xaxis[0].major_label_orientation = pi/8 - + plot.xaxis[0].major_label_orientation = pi / 8 def fill_boxplot( plot, source, prefix="", tooltips=None, tooltips_formatters=None, - js_tap_callback=None, server_tap_callback=None, + js_tap_callback=None, server_tap_callback=None ): # (Optional) Tooltip and tooltip formatters - if tooltips != None: - hover = HoverTool(tooltips = tooltips, mode="vline", names=["full_box"]) + if tooltips is not None: + hover = HoverTool(tooltips=tooltips, mode="vline", names=["full_box"]) - if tooltips_formatters != None: + if tooltips_formatters is not None: hover.formatters = tooltips_formatters plot.add_tools(hover) - # (Optional) Add TapTool (for JS tap callback) - if js_tap_callback != None: + if js_tap_callback is not None: tap = TapTool(callback=CustomJS(code=js_tap_callback)) plot.add_tools(tap) - # Draw boxes (the prefix argument modifies the fields of ColumnDataSource # that are used) @@ -128,18 +119,18 @@ def fill_boxplot( color="black" ) - # (Optional) Add server tap callback - if server_tap_callback != None: + if server_tap_callback is not None: top_stem.data_source.selected.on_change("indices", server_tap_callback) - bottom_stem.data_source.selected.on_change("indices", server_tap_callback) + bottom_stem.data_source.selected.on_change( + "indices", server_tap_callback) full_box.data_source.selected.on_change("indices", server_tap_callback) - bottom_box.data_source.selected.on_change("indices", server_tap_callback) + bottom_box.data_source.selected.on_change( + "indices", server_tap_callback) mu_dot.data_source.selected.on_change("indices", server_tap_callback) - # Plot appearance plot.xgrid.grid_line_color = None plot.ygrid.grid_line_color = None @@ -148,4 +139,4 @@ def fill_boxplot( plot.yaxis[0].formatter.power_limit_low = 0 plot.yaxis[0].formatter.precision = 3 - plot.xaxis[0].major_label_orientation = pi/8 + plot.xaxis[0].major_label_orientation = pi / 8 diff --git a/ci/vfc_ci_report/templates/index.html b/ci/vfc_ci_report/templates/index.html index 2ec7fc6..8fea8ec 100644 --- a/ci/vfc_ci_report/templates/index.html +++ b/ci/vfc_ci_report/templates/index.html @@ -291,19 +291,19 @@

Plots

-
+
{{ embed(roots.s_tabs_inspect) }}

-
+
{{ embed(roots.sigma_inspect) }}

-
+
{{ embed(roots.mu_inspect) }}
@@ -325,6 +325,8 @@
+ +