Chapter 4 Ranking

In this section, we will consider the Bradley-Terry Model for ranking algorithms in the fixed budget of 10,000 function evaluations per dimension and controlling for noise and the effect of benchmark functions

• RQ3: How can we rank algorithm different optimization algorithms given a budget of 10,000 evaluations per dimension in noisy benchmarks?

4.1 RQ3 Data preparation

We start importing the dataset

dataset <- readr::read_csv('./data/statscomp.csv')

The BT model formulation that we use has a specific data format, where we have one column with algo_0 (with index of each algorithm) another column with algo_1 and a third column with who won (algo 0 or algo 1),

First lets select only the data that we are interested and create ranking by the each run in each group (by the simNumber). To avoid ties (dealing with those on next session) we will rank ties randomly

d1 <- dataset %>% 
  dplyr::select(Algorithm, CostFunction, SD, Budget=MaxFevalPerDimensions, simNumber, TrueRewardDifference, OptimizationSuccessful) %>% 
  dplyr::filter(OptimizationSuccessful & Budget==10000 & SD==3.0) %>% 
  dplyr::select(-Budget, -OptimizationSuccessful, -SD) %>% 
  dplyr::group_by(CostFunction, simNumber) %>% 
  dplyr::mutate(rankReward=rank(TrueRewardDifference, ties.method = 'random')) %>% 
  dplyr::ungroup() %>% 
  dplyr::select(-TrueRewardDifference)

kable(dplyr::sample_n(d1,size=10), booktabs=T, format.args = list(scientific = FALSE), digits = 3) %>% 
  kable_styling(bootstrap_options = c("striped", "hover", "condensed")) %>% 
  kableExtra::scroll_box(width = "100%")

Algorithm	CostFunction	simNumber	rankReward
NelderMead	Trigonometric1N6	2	1
CMAES	DiscusN2	7	7
SimulatedAnnealing	ChenV	1	7
RandomSearch1	BentCigarN6	7	4
CMAES	Price1	8	6
CMAES	ChenV	2	6
CuckooSearch	Shubert	8	4
RandomSearch2	LunacekBiRastriginN6	1	5
SimulatedAnnealing	Mishra7N6	8	8
CMAES	StrechedVSineWave2N	8	3

Now to compare the ranks we need to pivot wider the data frame and based on that we will expand to the dataset in the appropriated format

d1_wide <- d1 %>% 
  tidyr::pivot_wider(names_from = Algorithm,
                     values_from=rankReward)

kable(dplyr::sample_n(d1_wide,size=10), booktabs=T, format.args = list(scientific = FALSE), digits = 3) %>% 
  kable_styling(bootstrap_options = c("striped", "hover", "condensed")) %>% 
  kableExtra::scroll_box(width = "100%")

CostFunction	simNumber	NelderMead	PSO	SimulatedAnnealing	CuckooSearch	DifferentialEvolution	RandomSearch1	RandomSearch2	CMAES
SphereN6	6	8	1	7	6	2	5	4	3
WhitleyN6	2	8	3	4	7	2	6	5	1
Schwefel2d4N6	8	8	3	7	6	2	4	5	1
Damavandi	9	8	6	7	1	2	4	3	5
Shubert	1	8	3	2	7	4	5	6	1
WhitleyN6	9	8	3	7	6	1	4	5	2
Mishra7N6	0	4	1	8	6	3	5	7	2
Damavandi	8	8	5	4	7	6	3	1	2
XinSheYang2N2	9	6	2	1	4	7	8	3	5
DiscusN2	9	8	3	1	6	4	2	5	7

Now we need to modify this data set and expand it so we have the pairwise comparisons

First let’s get the number of algorithms and create combination of all possible 2 by 2 comparisons without repeating

algorithms <- get_index_names_as_array(d1$Algorithm)
n_algorithms <- length(algorithms)
comb <- gtools::combinations(n=n_algorithms, r=2, v=seq(1:n_algorithms), repeats.allowed = F)

The pairs combinations looks like this (for algo_0 and algo_1):

kable(comb, booktabs=T) %>% 
  kable_styling(bootstrap_options = c("striped", "hover", "condensed")) %>% 
  kableExtra::scroll_box(width = "100%")

1	2
1	3
1	4
1	5
1	6
1	7
1	8
2	3
2	4
2	5
2	6
2	7
2	8
3	4
3	5
3	6
3	7
3	8
4	5
4	6
4	7
4	8
5	6
5	7
5	8
6	7
6	8
7	8

Note that each row of d_wide will be expanded into 28 rows. Giving a dataset with a total of 8400 rows.

The following code can a bit slow to run due to the double for loops (there is probably a way to vectorize this and make it run faster), but for building this appendix we will not run, instead we will run it once, save this data, and load it when needed. It takes a couple of minutes but if you have a lot of data and algorithms it can easily go for hours

We will use a progress bar to follow the data frame creation.

1- We initialize a tibble data frame 2- First we loop through the wide data frame d1_wide row by row 3- For each row we will loop through the different combinations in the comb variable to create the rows of the data frame. We add each row to the initial dataframe

pb <- progress::progress_bar$new(format = "[:bar] :current/:total (:percent)", total = nrow(d1_wide))

df_out <-  dplyr::tribble(~algo0_name, ~algo0, ~algo1_name, ~algo1, ~y, ~simNumber, ~CostFunction)

for(i in 1:nrow(d1_wide))
{
  current_row <- d1_wide[i,]
  for(j in 1:nrow(comb)){
    comb_row <- comb[j,]
    
    algo0_name <- algorithms[comb_row[1]]
    algo0 <- comb_row[1]
    algo0_rank <- current_row[[1,algo0_name]]
    
    algo1_name <- algorithms[comb_row[2]]
    algo1 <- comb_row[2]
    algo1_rank <- current_row[[1,algo1_name]]
    
    diff_rank <- algo1_rank - algo0_rank
    y <- ifelse(diff_rank<0, 1, 0) 
    
    
    df_out <- add_row(df_out,
                      algo0_name=algo0_name,
                      algo0=algo0,
                      algo1_name=algo1_name,
                      algo1=algo1,
                      y=y,
                      simNumber=current_row$simNumber,
                      CostFunction=current_row$CostFunction)
    
  }
  pb$tick()
}
saveRDS(df_out, file="./data/ranking.RDS")

Adding index for the benchmarks

df_out <- readRDS("./data/ranking.RDS")
df_out$CostFunctionId <- create_index(df_out$CostFunction)
benchmarks <- get_index_names_as_array(df_out$CostFunction)

Visualizing how the data frame looks like

kable(dplyr::sample_n(df_out,size=10), "html", booktabs=T, format.args = list(scientific = FALSE), digits = 3) %>% 
  kable_styling(bootstrap_options = c("striped", "hover", "condensed")) %>% 
  kableExtra::scroll_box(width = "100%")

algo0_name	algo0	algo1_name	algo1	y	simNumber	CostFunction	CostFunctionId
RandomSearch1	6	RandomSearch2	7	1	2	Trigonometric1N6	26
CMAES	1	PSO	5	0	2	ExponentialN2	7
CMAES	1	RandomSearch2	7	0	1	Schwefel2d20N2	16
CuckooSearch	2	RandomSearch1	6	1	9	QingN2	13
NelderMead	4	RandomSearch2	7	1	0	ChenBird	2
CMAES	1	PSO	5	1	7	ZakharovN2	30
NelderMead	4	RandomSearch1	6	1	0	ChungReynoldsN2	4
CuckooSearch	2	RandomSearch2	7	1	3	Schwefel2d23N6	18
DifferentialEvolution	3	SimulatedAnnealing	8	0	2	BentCigarN6	1
PSO	5	RandomSearch1	6	0	8	SalomonN2	15

4.2 RQ3 Stan model

The Stan model is specified in the file: './stanmodels/rankingmodel_withcluster.stan'.

print_stan_code('./stanmodels/rankingmodel_withcluster.stan')

// Ranking model with cluster data
// Author: David Issa Mattos
// Date: 1 Oct 2020
//
//

data {
 int <lower=1> N_total; // Sample size
 int y[N_total]; //variable that indicates which one wins algo0 oor algo 1

 int <lower=1> N_algorithm; // Number of algorithms
 
 int <lower=1> algo0[N_total];
 int <lower=1> algo1[N_total];
 
 // //To model the influence of each benchmark
 int <lower=1> N_bm;
 int bm_id[N_total];
}

parameters {
  real a_alg[N_algorithm]; //Latent variable that represents the strength value of each algorithm
  real<lower=0> s;//std for the random effects
  matrix[N_algorithm, N_bm] Uij; //parameters of the random effects for cluster
}

model {
  real p[N_total];
  a_alg ~ normal(0,2);
  s ~ exponential(0.1);
  for (i in 1:N_algorithm)
  {
    for(j in 1:N_bm){
       Uij[i, j] ~ normal(0, 1);
    }
   
  }

  for (i in 1:N_total)
  {
     p[i] = (a_alg[algo1[i]] + s*Uij[algo1[i], bm_id[i]]) - (a_alg[algo0[i]] + s*Uij[algo0[i], bm_id[i]] ) ;
  }
  
  y ~ bernoulli_logit(p);
}

Let’s compile and start sampling with the Stan function. In the data folder you can find the specific data used to fit the model after all transformations "./data/rankingmodel-withcluster-data.RDS"

For computation time sake we are not running this chunk every time we compile this document. From now on we will load from the saved Stan fit object. However, when we change our model or the data we can just run this chunk separately

standata <- list(
  N_total=nrow(df_out),
  y = as.integer(df_out$y),
  N_algorithm = length(algorithms),
  algo0=df_out$algo0,
  algo1=df_out$algo1,
  bm_id=df_out$CostFunctionId,
  N_bm=length(benchmarks)
  )
saveRDS(standata, file = "./data/rankingmodel-withcluster-data.RDS")
standata<-readRDS("./data/rankingmodel-withcluster-data.RDS")
ranking.fit <- stan(file = './stanmodels/rankingmodel_withcluster.stan',
                     data=standata,
                     chains = 4,
                     warmup = 200,
                     iter = 2000)
saveRDS(ranking.fit, file = "./data/ranking-with-cluster-fit.RDS")

ranking.fit <-readRDS("./data/ranking-with-cluster-fit.RDS")
a_alg <- c("a_alg[1]",
           "a_alg[2]",
           "a_alg[3]",
           "a_alg[4]",
           "a_alg[5]",
           "a_alg[6]",
           "a_alg[7]",
           "a_alg[8]")

4.3 RQ3 Diagnosis

rstan::traceplot(ranking.fit, pars=c(a_alg,'s'))

Another diagnosis is to look at the Rhat. If Rhat is greater than 1.05 it indicates a divergence in the chains (they did not mix well). The table below shows a summary of the posteriors. Note that we have several random effects parameter estimates.

kable(summary(ranking.fit)$summary) %>% 
  kable_styling(bootstrap_options = c('striped',"hover", "condensed" )) %>% 
  kableExtra::scroll_box(width = "100%")

	mean	se_mean	sd	2.5%	25%	50%	75%	97.5%	n_eff	Rhat
a_alg[1]	1.0350990	0.0099375	0.7818263	-0.4889451	0.5245045	1.0272608	1.5465781	2.5859266	6189.692	0.9998847
a_alg[2]	-0.4086096	0.0097442	0.7778534	-1.9583610	-0.9293837	-0.4082742	0.1144884	1.1325767	6372.434	0.9996527
a_alg[3]	1.9855326	0.0101607	0.7833351	0.4453243	1.4707039	1.9837454	2.5001817	3.5373887	5943.627	0.9997423
a_alg[4]	-2.9812765	0.0103527	0.7967958	-4.5108464	-3.5157775	-2.9724022	-2.4480854	-1.4303523	5923.626	1.0001171
a_alg[5]	1.5786103	0.0100043	0.7871080	0.0462318	1.0553736	1.5831165	2.0952817	3.1108493	6190.105	0.9997322
a_alg[6]	0.2820188	0.0099938	0.7906221	-1.2662524	-0.2510642	0.2702926	0.8240528	1.8092083	6258.569	0.9998661
a_alg[7]	0.2511935	0.0098370	0.7876031	-1.2844233	-0.2782489	0.2510030	0.7890238	1.7830516	6410.446	0.9998134
a_alg[8]	-1.6873176	0.0100228	0.7843961	-3.2354333	-2.2125968	-1.6899210	-1.1695586	-0.1369124	6124.853	0.9997644
s	1.8186475	0.0026735	0.1185007	1.5974726	1.7367581	1.8133418	1.8958155	2.0619577	1964.622	1.0014523
Uij[1,1]	2.7568246	0.0068488	0.6168957	1.5905147	2.3314248	2.7379874	3.1716803	4.0020500	8113.306	0.9999433
Uij[1,2]	-2.0117811	0.0054712	0.4613464	-2.9433107	-2.3196997	-1.9983697	-1.6997434	-1.1434574	7110.256	0.9998827
Uij[1,3]	-1.0160761	0.0048771	0.4231998	-1.8436816	-1.2967851	-1.0130569	-0.7369512	-0.1841409	7529.647	1.0000163
Uij[1,4]	0.3563913	0.0048953	0.4326111	-0.4691134	0.0613761	0.3507387	0.6504564	1.2140202	7809.887	0.9998143
Uij[1,5]	-0.1472215	0.0046975	0.4254996	-0.9748673	-0.4345510	-0.1545260	0.1406149	0.6839556	8204.801	1.0001625
Uij[1,6]	-2.3973034	0.0057218	0.5129544	-3.4075592	-2.7420730	-2.3964285	-2.0419191	-1.4212910	8037.052	1.0008644
Uij[1,7]	-1.4078985	0.0049489	0.4284437	-2.2523282	-1.6954565	-1.4018171	-1.1196755	-0.5791312	7495.086	1.0005379
Uij[1,8]	-0.7674291	0.0050325	0.4228531	-1.6081325	-1.0460228	-0.7639979	-0.4808151	0.0586314	7060.226	0.9997273
Uij[1,9]	1.1122770	0.0052789	0.4955415	0.1426358	0.7797746	1.1032002	1.4400893	2.1038175	8812.076	1.0002491
Uij[1,10]	1.3826008	0.0056059	0.4816300	0.4669824	1.0563422	1.3832680	1.7059654	2.3426203	7381.241	1.0003584
Uij[1,11]	1.0280591	0.0051854	0.4890837	0.0807931	0.6962308	1.0149521	1.3558423	2.0016199	8896.151	0.9998362
Uij[1,12]	-0.3671331	0.0049811	0.4098012	-1.1727494	-0.6382965	-0.3626950	-0.0948469	0.4406861	6768.464	1.0000148
Uij[1,13]	0.7401377	0.0051266	0.4382564	-0.0975171	0.4425075	0.7363724	1.0347060	1.6099385	7307.848	1.0004634
Uij[1,14]	1.5977910	0.0051860	0.4924602	0.6419273	1.2637216	1.5975200	1.9360789	2.5583447	9017.419	0.9998956
Uij[1,15]	-2.8218921	0.0064887	0.5298789	-3.8887420	-3.1737296	-2.8081934	-2.4572131	-1.8102215	6668.585	1.0001931
Uij[1,16]	-0.3449782	0.0049085	0.4315929	-1.1923280	-0.6397734	-0.3394316	-0.0544866	0.4954096	7731.414	1.0001765
Uij[1,17]	-0.2023738	0.0048264	0.4312314	-1.0474327	-0.4889775	-0.1996496	0.0878083	0.6298177	7983.114	1.0000599
Uij[1,18]	1.4790742	0.0051569	0.4860841	0.5489460	1.1450392	1.4778527	1.8072492	2.4212830	8884.640	0.9996503
Uij[1,19]	1.1583333	0.0055574	0.4782449	0.2307281	0.8415235	1.1501705	1.4766778	2.0871754	7405.594	0.9997612
Uij[1,20]	2.4816319	0.0067180	0.5918140	1.3702240	2.0746444	2.4577437	2.8679558	3.6880249	7760.453	1.0000875
Uij[1,21]	-0.1058211	0.0046657	0.4166140	-0.9158034	-0.3822645	-0.1072440	0.1680613	0.7142994	7973.206	0.9996953
Uij[1,22]	0.7798184	0.0050579	0.4805069	-0.1575611	0.4546586	0.7763659	1.1039509	1.7378154	9025.204	0.9999745
Uij[1,23]	-0.1487943	0.0049382	0.4198536	-0.9823316	-0.4372502	-0.1484105	0.1368763	0.6776679	7228.541	0.9996262
Uij[1,24]	-0.6146686	0.0046842	0.4090193	-1.4306410	-0.8911070	-0.6109428	-0.3324223	0.1675987	7624.646	1.0001753
Uij[1,25]	-0.8634452	0.0050265	0.4174748	-1.6889816	-1.1475820	-0.8666918	-0.5766713	-0.0342703	6898.128	0.9999936
Uij[1,26]	-0.5845752	0.0046152	0.4126612	-1.3717355	-0.8738679	-0.5869563	-0.3024878	0.2312067	7994.861	1.0003718
Uij[1,27]	-1.5679692	0.0051020	0.4514158	-2.4771515	-1.8722151	-1.5655713	-1.2647471	-0.7010626	7828.501	1.0005969
Uij[1,28]	1.6374628	0.0054645	0.5310866	0.6053352	1.2810035	1.6363277	1.9869884	2.7069031	9445.681	0.9998314
Uij[1,29]	-0.3994496	0.0046884	0.4095300	-1.2048221	-0.6695395	-0.4006865	-0.1184461	0.4011749	7629.896	0.9998467
Uij[1,30]	-0.2747694	0.0049231	0.4199378	-1.0941249	-0.5581011	-0.2793798	0.0090081	0.5555291	7275.939	0.9997294
Uij[2,1]	-0.9118255	0.0050621	0.4614396	-1.8303036	-1.2250132	-0.9132414	-0.5948358	-0.0307307	8309.262	1.0002471
Uij[2,2]	0.7360394	0.0048665	0.4189589	-0.0820554	0.4524802	0.7333332	1.0246713	1.5382982	7411.590	0.9995565
Uij[2,3]	0.1855127	0.0048088	0.4110061	-0.6349756	-0.0859329	0.1803901	0.4523463	1.0080215	7305.198	0.9995848
Uij[2,4]	-0.1561007	0.0052024	0.4522417	-1.0472026	-0.4601719	-0.1544331	0.1495431	0.7313382	7556.670	1.0000256
Uij[2,5]	0.5686377	0.0046033	0.4241218	-0.2657993	0.2837517	0.5759036	0.8515573	1.3920251	8488.576	0.9999016
Uij[2,6]	-0.2454198	0.0049806	0.4431801	-1.0994024	-0.5538180	-0.2392124	0.0499348	0.6258264	7917.615	1.0002062
Uij[2,7]	0.3005368	0.0047756	0.4116191	-0.5117091	0.0234118	0.3024190	0.5774570	1.1091651	7429.163	1.0007859
Uij[2,8]	0.1038937	0.0049971	0.4161380	-0.7240727	-0.1731505	0.1094736	0.3813164	0.9073294	6934.755	1.0001131
Uij[2,9]	-0.6147997	0.0049292	0.4569832	-1.5164804	-0.9174888	-0.6101217	-0.3046565	0.2881951	8595.067	1.0001000
Uij[2,10]	-0.0839954	0.0047354	0.4267774	-0.9161901	-0.3756604	-0.0886199	0.2093155	0.7524626	8122.392	1.0000538
Uij[2,11]	-0.4093252	0.0049963	0.4525534	-1.3046255	-0.7126732	-0.4082073	-0.1030230	0.4635166	8204.379	1.0000034
Uij[2,12]	0.3225484	0.0048765	0.4059902	-0.4764681	0.0408683	0.3262363	0.6025226	1.0987600	6931.354	1.0005904
Uij[2,13]	0.3573765	0.0051299	0.4326365	-0.4866159	0.0707737	0.3597662	0.6501994	1.1939192	7112.531	1.0000188
Uij[2,14]	-0.2901255	0.0048619	0.4497415	-1.1704280	-0.6003112	-0.2920998	0.0169833	0.5864769	8556.787	1.0000544
Uij[2,15]	0.7022558	0.0049771	0.4222572	-0.1150877	0.4195670	0.7010846	0.9836285	1.5317844	7197.845	0.9997664
Uij[2,16]	0.6945807	0.0049894	0.4272260	-0.1328749	0.4047755	0.6916793	0.9795240	1.5469742	7331.952	0.9998481
Uij[2,17]	-0.2185615	0.0048720	0.4382104	-1.0984592	-0.5134980	-0.2113982	0.0781627	0.6276812	8090.155	1.0001446
Uij[2,18]	-0.3206384	0.0049597	0.4692944	-1.2375007	-0.6393195	-0.3115535	-0.0046453	0.6043356	8953.384	0.9997829
Uij[2,19]	0.4651950	0.0046032	0.4258835	-0.3668718	0.1747166	0.4626164	0.7540787	1.2940660	8559.786	0.9998108
Uij[2,20]	-0.9697870	0.0048787	0.4669374	-1.9028781	-1.2808106	-0.9648317	-0.6616987	-0.0596379	9160.448	1.0001722
Uij[2,21]	0.0559637	0.0047342	0.4247027	-0.7753249	-0.2333481	0.0550506	0.3469145	0.8794904	8047.959	0.9996142
Uij[2,22]	-1.3275595	0.0051314	0.4840651	-2.2914464	-1.6502408	-1.3208356	-1.0067086	-0.3925727	8899.026	0.9995716
Uij[2,23]	0.0839835	0.0049526	0.4195062	-0.7455468	-0.2031012	0.0874359	0.3604104	0.9212649	7174.795	1.0000113
Uij[2,24]	0.1323553	0.0048279	0.4103965	-0.6649977	-0.1476288	0.1315250	0.4118975	0.9341050	7225.840	1.0000912
Uij[2,25]	0.3719959	0.0049697	0.4207036	-0.4631670	0.0958649	0.3710583	0.6513204	1.1997728	7166.234	0.9997759
Uij[2,26]	-0.2964895	0.0045362	0.4157191	-1.1284417	-0.5798734	-0.3012985	-0.0138295	0.5166549	8398.689	0.9999439
Uij[2,27]	0.8768972	0.0049258	0.4350635	0.0285952	0.5838022	0.8779145	1.1687862	1.7280956	7801.049	1.0003535
Uij[2,28]	-0.8093466	0.0049757	0.4614552	-1.7218763	-1.1187694	-0.8034172	-0.4947902	0.0785906	8601.011	0.9998047
Uij[2,29]	0.0758406	0.0046521	0.4096718	-0.7413744	-0.1903349	0.0764793	0.3478485	0.8696867	7754.897	1.0005194
Uij[2,30]	0.4084189	0.0048212	0.4170506	-0.4126051	0.1222951	0.4151572	0.6837595	1.2171550	7482.812	0.9996785
Uij[3,1]	1.0073477	0.0056024	0.5272701	-0.0070510	0.6585322	0.9983542	1.3568953	2.0481410	8857.636	1.0004760
Uij[3,2]	0.4494440	0.0055275	0.4612767	-0.4445557	0.1406110	0.4484923	0.7448691	1.3775233	6964.002	0.9998257
Uij[3,3]	-1.6023314	0.0050154	0.4282851	-2.4575609	-1.8902213	-1.6014886	-1.3103747	-0.7596825	7292.256	0.9996174
Uij[3,4]	0.3180840	0.0047771	0.4399761	-0.5296537	0.0255057	0.3126647	0.6131122	1.1813983	8482.623	1.0000645
Uij[3,5]	-0.6641079	0.0045801	0.4269222	-1.4850328	-0.9579079	-0.6660101	-0.3689882	0.1608834	8688.658	0.9999617
Uij[3,6]	0.2326513	0.0050891	0.4413233	-0.6207068	-0.0653579	0.2305346	0.5307241	1.1156567	7520.371	1.0001218
Uij[3,7]	-0.9653100	0.0047645	0.4100695	-1.7632959	-1.2479738	-0.9651005	-0.6912685	-0.1526064	7407.724	1.0001151
Uij[3,8]	-1.2842741	0.0052116	0.4172326	-2.1196463	-1.5598092	-1.2809310	-1.0027277	-0.4882449	6409.460	0.9996239
Uij[3,9]	0.5335109	0.0051922	0.4905067	-0.4254207	0.2079019	0.5362266	0.8650988	1.4927805	8924.692	0.9996950
Uij[3,10]	0.1541278	0.0050599	0.4510499	-0.7386530	-0.1543444	0.1553414	0.4548776	1.0559184	7946.308	1.0004800
Uij[3,11]	1.6121375	0.0058877	0.5383345	0.5858994	1.2456417	1.5952324	1.9730634	2.7112403	8360.260	1.0000395
Uij[3,12]	-1.0101996	0.0048170	0.4106617	-1.8339218	-1.2856831	-1.0116214	-0.7348951	-0.2074507	7268.152	1.0000251
Uij[3,13]	-0.5352153	0.0051829	0.4322857	-1.3682473	-0.8216571	-0.5326458	-0.2435968	0.3136118	6956.658	1.0005153
Uij[3,14]	1.2543606	0.0054120	0.4888469	0.3119591	0.9204890	1.2494931	1.5864753	2.2339286	8158.870	1.0001832
Uij[3,15]	-0.2721445	0.0049936	0.4223608	-1.0855721	-0.5648840	-0.2729688	0.0089531	0.5674364	7153.844	1.0003756
Uij[3,16]	-0.0410410	0.0047780	0.4303627	-0.8665422	-0.3387241	-0.0507189	0.2497476	0.8206513	8112.979	0.9998610
Uij[3,17]	1.2865672	0.0062544	0.5398753	0.2606436	0.9166113	1.2718928	1.6461927	2.3826536	7451.016	0.9995780
Uij[3,18]	1.0438479	0.0053805	0.4852059	0.0991268	0.7090153	1.0447925	1.3705275	1.9849483	8132.276	0.9999111
Uij[3,19]	1.0255994	0.0056218	0.4974867	0.0639425	0.6872134	1.0176544	1.3561308	1.9942321	7830.821	1.0000818
Uij[3,20]	0.5705030	0.0051968	0.4763676	-0.3519639	0.2448030	0.5655583	0.8977213	1.5109338	8402.703	0.9998861
Uij[3,21]	-0.0469374	0.0050064	0.4346849	-0.8891189	-0.3387192	-0.0530623	0.2465612	0.8252608	7538.760	0.9997205
Uij[3,22]	0.8907837	0.0054451	0.4955141	-0.0686155	0.5476846	0.8889436	1.2243206	1.8669988	8281.294	0.9996579
Uij[3,23]	-0.8606064	0.0048879	0.4111195	-1.6729920	-1.1410176	-0.8582076	-0.5856793	-0.0586748	7074.423	0.9998325
Uij[3,24]	-0.9155754	0.0047352	0.4078674	-1.7183378	-1.1856843	-0.9162233	-0.6417720	-0.1151063	7419.371	1.0001275
Uij[3,25]	-0.7202299	0.0050933	0.4212868	-1.5601059	-1.0034157	-0.7171780	-0.4375285	0.1005430	6841.499	0.9999412
Uij[3,26]	-0.9790499	0.0045789	0.4110545	-1.7859790	-1.2523830	-0.9781484	-0.7031598	-0.1634507	8058.986	1.0000878
Uij[3,27]	0.2145017	0.0048992	0.4430642	-0.6478430	-0.0804793	0.2087423	0.5071635	1.0929758	8178.624	1.0004294
Uij[3,28]	2.0605100	0.0062671	0.5785991	0.9638991	1.6584069	2.0490457	2.4419682	3.2332636	8523.600	0.9999511
Uij[3,29]	-1.3502330	0.0047122	0.4112604	-2.1525129	-1.6270620	-1.3519898	-1.0796666	-0.5433599	7616.959	0.9999308
Uij[3,30]	-0.5482071	0.0048715	0.4214139	-1.3656972	-0.8377365	-0.5534726	-0.2540713	0.2871019	7483.374	0.9998348
Uij[4,1]	-1.0547976	0.0057939	0.5258410	-2.1423295	-1.4061579	-1.0444071	-0.7014771	-0.0599811	8237.012	0.9997875
Uij[4,2]	1.4152369	0.0050122	0.4222379	0.5847297	1.1320818	1.4190580	1.7011842	2.2436516	7096.794	0.9996116
Uij[4,3]	1.2888832	0.0051489	0.4264335	0.4681648	0.9979293	1.2831645	1.5828093	2.1267815	6859.112	0.9995560
Uij[4,4]	-0.2175562	0.0055988	0.5128904	-1.2512756	-0.5593706	-0.2155310	0.1363785	0.7734667	8391.771	0.9999658
Uij[4,5]	-0.9999722	0.0082125	0.6873597	-2.4298974	-1.4415019	-0.9586839	-0.5131495	0.2527938	7005.226	0.9998558
Uij[4,6]	-0.3850099	0.0054081	0.5058200	-1.3978102	-0.7159066	-0.3739554	-0.0426153	0.5630651	8747.817	0.9997977
Uij[4,7]	2.2690834	0.0051886	0.4365670	1.4157615	1.9724727	2.2711768	2.5615590	3.1315005	7079.385	1.0000048
Uij[4,8]	1.1511079	0.0051838	0.4241700	0.3032800	0.8721010	1.1546695	1.4351798	1.9730284	6695.441	1.0006260
Uij[4,9]	-1.4868073	0.0079324	0.6467099	-2.8417259	-1.8958611	-1.4631879	-1.0382503	-0.3124344	6646.675	0.9999836
Uij[4,10]	0.8863267	0.0052023	0.4409030	0.0126595	0.5910444	0.8893569	1.1766049	1.7544285	7182.722	1.0005518
Uij[4,11]	-1.5339463	0.0077762	0.6502043	-2.9039419	-1.9486211	-1.5010588	-1.0846510	-0.3422617	6991.480	1.0001814
Uij[4,12]	0.2032391	0.0055015	0.4590260	-0.6980710	-0.1016079	0.2084355	0.5165290	1.0649153	6961.560	1.0001354
Uij[4,13]	-0.3925842	0.0060215	0.5415076	-1.4888802	-0.7477202	-0.3743811	-0.0213750	0.6247565	8087.330	1.0003910
Uij[4,14]	-1.0686745	0.0057736	0.5331029	-2.1423804	-1.4144230	-1.0586174	-0.7113862	-0.0299121	8525.592	1.0000143
Uij[4,15]	0.5335171	0.0050969	0.4502252	-0.3564267	0.2312813	0.5306487	0.8325837	1.4213166	7802.747	1.0003882
Uij[4,16]	-0.5016088	0.0057299	0.5521439	-1.6211547	-0.8615124	-0.4936333	-0.1214575	0.5467288	9285.621	0.9999972
Uij[4,17]	0.1705628	0.0054261	0.4751121	-0.7500838	-0.1530714	0.1707272	0.5002412	1.0745915	7666.979	0.9995502
Uij[4,18]	-0.5736415	0.0055927	0.5318950	-1.6265162	-0.9343312	-0.5703938	-0.2000789	0.4496235	9044.941	0.9998754
Uij[4,19]	0.2321728	0.0049765	0.4541364	-0.6816344	-0.0712835	0.2388763	0.5432059	1.1044883	8327.613	0.9997770
Uij[4,20]	-1.6276982	0.0070903	0.6298337	-2.9424722	-2.0267361	-1.6110177	-1.1951056	-0.4638435	7890.771	1.0001433
Uij[4,21]	-1.0568413	0.0082121	0.6954488	-2.5205506	-1.4852024	-1.0267179	-0.5783466	0.2099793	7171.753	0.9999914
Uij[4,22]	-1.6012507	0.0063108	0.5688484	-2.7643341	-1.9668914	-1.5925988	-1.2223617	-0.5185397	8125.108	0.9997419
Uij[4,23]	0.8565832	0.0050033	0.4290898	0.0053646	0.5724052	0.8574249	1.1540984	1.6820064	7355.129	1.0002642
Uij[4,24]	0.5869816	0.0051309	0.4326763	-0.2741647	0.2885961	0.5883067	0.8834859	1.4277965	7111.227	0.9999143
Uij[4,25]	0.3390502	0.0052386	0.4531598	-0.5562876	0.0367418	0.3431145	0.6411253	1.2120223	7482.901	0.9996849
Uij[4,26]	1.4815727	0.0049017	0.4226446	0.6749818	1.1930830	1.4713979	1.7658538	2.3244273	7434.608	1.0003777
Uij[4,27]	0.1659433	0.0051940	0.4600652	-0.7582899	-0.1361634	0.1733586	0.4785447	1.0384514	7845.866	1.0000199
Uij[4,28]	-1.2806149	0.0065112	0.5706343	-2.4592248	-1.6526341	-1.2604375	-0.8855718	-0.1926687	7680.498	1.0004478
Uij[4,29]	0.8940605	0.0048851	0.4256570	0.0509766	0.6162184	0.8931302	1.1736058	1.7256979	7592.286	1.0009941
Uij[4,30]	-0.0123213	0.0056647	0.4860611	-0.9555250	-0.3465494	-0.0018815	0.3236089	0.9059362	7362.537	0.9997726
Uij[5,1]	0.1610819	0.0054275	0.4909976	-0.8066471	-0.1726553	0.1590742	0.4973047	1.1202438	8183.943	1.0000650
Uij[5,2]	-0.6879136	0.0049420	0.4074034	-1.4862566	-0.9626568	-0.6935778	-0.4132975	0.1100455	6795.886	0.9996164
Uij[5,3]	-0.0763661	0.0049768	0.4293301	-0.9076048	-0.3660849	-0.0797143	0.2124873	0.7678842	7441.954	0.9997382
Uij[5,4]	0.2008752	0.0049600	0.4412798	-0.6542883	-0.1016702	0.2010888	0.4999876	1.0749466	7915.213	1.0001994
Uij[5,5]	-0.3791894	0.0043556	0.4237388	-1.2022060	-0.6605014	-0.3786616	-0.0887175	0.4374964	9464.666	0.9997955
Uij[5,6]	-0.1396903	0.0048631	0.4298054	-0.9736264	-0.4308501	-0.1412236	0.1514906	0.7022374	7811.186	1.0004841
Uij[5,7]	-1.1925048	0.0049356	0.4194588	-2.0305877	-1.4782757	-1.1897950	-0.9138555	-0.3823593	7222.731	1.0006622
Uij[5,8]	-0.8229707	0.0049344	0.4154215	-1.6529174	-1.0997631	-0.8125020	-0.5490555	-0.0065503	7087.623	0.9999104
Uij[5,9]	1.6620525	0.0055731	0.5287226	0.6613757	1.2985690	1.6572067	2.0100333	2.7178656	9000.311	1.0000451
Uij[5,10]	0.5745866	0.0053482	0.4564669	-0.3198793	0.2746407	0.5730956	0.8819339	1.4688604	7284.481	0.9997806
Uij[5,11]	0.7421610	0.0051307	0.4834058	-0.2162685	0.4145210	0.7401900	1.0622076	1.7063505	8877.106	0.9997100
Uij[5,12]	-0.4984991	0.0048439	0.4109265	-1.3099250	-0.7759348	-0.4925025	-0.2199153	0.2952362	7196.885	1.0002638
Uij[5,13]	0.0104689	0.0050218	0.4268828	-0.8355194	-0.2790088	0.0134855	0.3011566	0.8343938	7225.920	1.0001855
Uij[5,14]	0.7068502	0.0051046	0.4768913	-0.2217838	0.3906000	0.6988112	1.0216474	1.6436442	8728.075	1.0000220
Uij[5,15]	0.3028525	0.0049623	0.4293768	-0.5440520	0.0162137	0.3035211	0.5943020	1.1275868	7486.961	0.9997411
Uij[5,16]	0.6871776	0.0049884	0.4377777	-0.1735442	0.3843984	0.6889344	0.9892943	1.5299438	7701.531	0.9997284
Uij[5,17]	-0.2144385	0.0048584	0.4336255	-1.0704499	-0.5107113	-0.2090975	0.0789814	0.6316863	7965.900	1.0001001
Uij[5,18]	1.1159882	0.0049244	0.4791961	0.1927222	0.7980919	1.1176990	1.4388016	2.0453534	9469.342	0.9997376
Uij[5,19]	-1.2582857	0.0048452	0.4314994	-2.1236219	-1.5442036	-1.2615177	-0.9687659	-0.4378307	7931.232	0.9998592
Uij[5,20]	-0.2284588	0.0045918	0.4491880	-1.1084821	-0.5317220	-0.2309842	0.0753132	0.6530863	9569.604	0.9998917
Uij[5,21]	-0.2624154	0.0046866	0.4258832	-1.0855067	-0.5432064	-0.2705560	0.0211492	0.5852048	8257.903	0.9997635
Uij[5,22]	2.5212396	0.0070576	0.6089231	1.3708082	2.1078268	2.5018016	2.9109950	3.7519946	7444.068	0.9998780
Uij[5,23]	-0.4108674	0.0049068	0.4091119	-1.2175339	-0.6914275	-0.4069610	-0.1268971	0.3721501	6951.642	0.9995629
Uij[5,24]	-0.7564929	0.0045936	0.4066680	-1.5624618	-1.0303299	-0.7567261	-0.4871905	0.0455760	7837.313	0.9998797
Uij[5,25]	-0.3547424	0.0050576	0.4265396	-1.1826898	-0.6438791	-0.3546845	-0.0677634	0.4885426	7112.606	0.9997734
Uij[5,26]	-0.5484119	0.0046012	0.4103649	-1.3405153	-0.8286629	-0.5513117	-0.2747002	0.2700166	7954.267	1.0006088
Uij[5,27]	0.2238965	0.0051198	0.4396453	-0.6290873	-0.0772777	0.2267578	0.5168140	1.1079665	7373.867	1.0004717
Uij[5,28]	0.2166104	0.0051487	0.4884776	-0.7501314	-0.1182536	0.2132065	0.5396021	1.1746844	9001.187	0.9998080
Uij[5,29]	-0.2949915	0.0046411	0.4090049	-1.1099320	-0.5640623	-0.2939212	-0.0247612	0.5016446	7766.342	0.9999574
Uij[5,30]	-0.2160241	0.0048692	0.4178932	-1.0203039	-0.5016691	-0.2152328	0.0640756	0.5903540	7365.686	1.0002223
Uij[6,1]	-0.6247991	0.0050568	0.4536326	-1.5268210	-0.9293660	-0.6256297	-0.3168351	0.2622814	8047.318	1.0000593
Uij[6,2]	-0.4187735	0.0048960	0.4103144	-1.2329170	-0.6884198	-0.4175403	-0.1399352	0.3985669	7023.360	0.9996310
Uij[6,3]	0.0985780	0.0051401	0.4209715	-0.7269067	-0.1840676	0.1003228	0.3809159	0.9276387	6707.564	0.9995862
Uij[6,4]	0.2659752	0.0049877	0.4436955	-0.6081839	-0.0374348	0.2662519	0.5762164	1.1168504	7913.396	1.0002142
Uij[6,5]	0.0299219	0.0046911	0.4241500	-0.7971327	-0.2598001	0.0291514	0.3223569	0.8465939	8175.147	1.0002137
Uij[6,6]	0.6451811	0.0048321	0.4292219	-0.1860810	0.3565973	0.6461714	0.9270589	1.4933868	7890.377	1.0003926
Uij[6,7]	0.2538168	0.0046767	0.4086395	-0.5461182	-0.0309398	0.2551826	0.5362982	1.0505904	7634.826	1.0003590
Uij[6,8]	0.4274660	0.0049731	0.4179161	-0.4025759	0.1493242	0.4315904	0.7056309	1.2356792	7061.842	1.0000938
Uij[6,9]	-0.5395962	0.0046649	0.4501384	-1.4029731	-0.8487781	-0.5398695	-0.2310737	0.3404310	9311.042	1.0002101
Uij[6,10]	-0.8084559	0.0048717	0.4339794	-1.6550548	-1.1022997	-0.8044388	-0.5155709	0.0343890	7935.489	1.0003847
Uij[6,11]	-0.6109773	0.0048729	0.4495825	-1.4976911	-0.9182983	-0.6088736	-0.3072446	0.2606366	8512.265	0.9999330
Uij[6,12]	-0.1475828	0.0049173	0.4079584	-0.9443651	-0.4288301	-0.1429433	0.1330495	0.6301994	6882.900	0.9998486
Uij[6,13]	0.7110351	0.0050460	0.4275339	-0.1339759	0.4235922	0.7136160	0.9927999	1.5569792	7178.797	1.0007566
Uij[6,14]	-0.1825220	0.0049111	0.4478515	-1.0738590	-0.4814001	-0.1794772	0.1207994	0.6942991	8316.060	1.0008052
Uij[6,15]	0.8600299	0.0052773	0.4319325	0.0020161	0.5668183	0.8583955	1.1470909	1.7159696	6698.982	0.9998806
Uij[6,16]	0.5314289	0.0049489	0.4213566	-0.2937868	0.2466523	0.5299128	0.8109240	1.3717170	7249.108	0.9998220
Uij[6,17]	0.1982816	0.0047830	0.4284962	-0.6233876	-0.0986099	0.1980872	0.4916411	1.0310962	8025.852	1.0001748
Uij[6,18]	-0.0586963	0.0049996	0.4614932	-0.9639492	-0.3712287	-0.0581867	0.2532450	0.8373281	8520.296	0.9996152
Uij[6,19]	-0.0080057	0.0049024	0.4306798	-0.8471121	-0.2934495	-0.0079689	0.2831372	0.8383824	7717.910	1.0000249
Uij[6,20]	0.2211151	0.0048568	0.4510694	-0.6568711	-0.0825393	0.2210293	0.5232522	1.0989568	8625.567	0.9999425
Uij[6,21]	-0.0495386	0.0047408	0.4168294	-0.8652290	-0.3299256	-0.0516559	0.2319948	0.7689865	7730.764	0.9997201
Uij[6,22]	-0.3717223	0.0053212	0.4680513	-1.2991666	-0.6835655	-0.3701338	-0.0622979	0.5494174	7737.028	0.9996392
Uij[6,23]	-0.3540491	0.0048657	0.4145042	-1.1688097	-0.6318058	-0.3490797	-0.0734128	0.4408301	7257.065	0.9999383
Uij[6,24]	0.1959043	0.0048074	0.4080546	-0.6184873	-0.0689125	0.1926992	0.4701189	1.0018415	7204.835	0.9996239
Uij[6,25]	0.4137847	0.0050166	0.4231668	-0.4038988	0.1272633	0.4125788	0.7014688	1.2340569	7115.622	0.9996148
Uij[6,26]	-0.2363742	0.0045652	0.4101442	-1.0430403	-0.5183638	-0.2328827	0.0350588	0.5735567	8071.441	1.0000106
Uij[6,27]	0.5553174	0.0049240	0.4363689	-0.2960677	0.2593486	0.5495635	0.8525548	1.4199907	7853.639	1.0000431
Uij[6,28]	-0.4486609	0.0048792	0.4598944	-1.3344911	-0.7507847	-0.4533277	-0.1409905	0.4652987	8884.061	0.9996815
Uij[6,29]	-0.2704751	0.0047704	0.4091097	-1.0781167	-0.5409605	-0.2680438	-0.0000097	0.5355939	7354.713	1.0005713
Uij[6,30]	-0.1030006	0.0048923	0.4204442	-0.9290630	-0.3878017	-0.1069345	0.1813979	0.7205199	7385.697	0.9998357
Uij[7,1]	-0.6686367	0.0050088	0.4557680	-1.5845243	-0.9699501	-0.6700753	-0.3655302	0.2316460	8279.956	1.0004251
Uij[7,2]	-0.2566409	0.0049552	0.4096727	-1.0817867	-0.5276101	-0.2558151	0.0157505	0.5481599	6835.313	0.9997654
Uij[7,3]	0.2504599	0.0049710	0.4217731	-0.5675008	-0.0370700	0.2473867	0.5333353	1.0803985	7199.055	0.9998415
Uij[7,4]	0.6859212	0.0050999	0.4419246	-0.1685289	0.3834376	0.6864736	0.9898366	1.5408539	7508.894	0.9999479
Uij[7,5]	0.5750226	0.0045292	0.4292228	-0.2560126	0.2836859	0.5738421	0.8653033	1.4180462	8980.797	1.0000379
Uij[7,6]	0.5765717	0.0049524	0.4323608	-0.2676764	0.2743627	0.5737352	0.8682920	1.4216896	7621.919	1.0007105
Uij[7,7]	0.1360901	0.0049709	0.4130857	-0.6627489	-0.1475640	0.1364560	0.4186504	0.9491536	6905.786	1.0008082
Uij[7,8]	0.1086326	0.0050418	0.4155035	-0.7096056	-0.1769173	0.1150681	0.3988567	0.9075658	6791.593	0.9996560
Uij[7,9]	-0.6912170	0.0049313	0.4570350	-1.5732257	-1.0027622	-0.6937236	-0.3799759	0.1878556	8589.708	1.0003142
Uij[7,10]	-1.2988127	0.0051996	0.4439050	-2.1663917	-1.6026462	-1.2937276	-0.9931531	-0.4350331	7288.674	1.0002294
Uij[7,11]	-0.6481199	0.0051177	0.4543147	-1.5596388	-0.9505406	-0.6381087	-0.3428891	0.2293004	7880.840	1.0000041
Uij[7,12]	0.2787110	0.0048952	0.4131514	-0.5265008	-0.0000545	0.2783038	0.5566697	1.0958326	7123.327	1.0001954
Uij[7,13]	0.6035767	0.0051274	0.4296069	-0.2423933	0.3187230	0.6027932	0.8956915	1.4443235	7020.208	1.0003450
Uij[7,14]	-0.7790838	0.0049209	0.4487950	-1.6773649	-1.0776840	-0.7779394	-0.4793608	0.1081412	8317.766	1.0002652
Uij[7,15]	0.6962269	0.0049467	0.4255781	-0.1138881	0.4092854	0.6967696	0.9822225	1.5231681	7401.711	0.9999640
Uij[7,16]	0.3818410	0.0048435	0.4306669	-0.4638307	0.0867805	0.3875423	0.6719341	1.2334873	7906.206	0.9999178
Uij[7,17]	0.3570449	0.0048365	0.4263812	-0.4960780	0.0792652	0.3549124	0.6429293	1.1736216	7771.964	1.0005065
Uij[7,18]	-0.4615351	0.0048152	0.4633419	-1.3737511	-0.7695673	-0.4553099	-0.1494646	0.4327476	9259.286	0.9995530
Uij[7,19]	-0.2435080	0.0047296	0.4275416	-1.0939372	-0.5311344	-0.2456423	0.0485947	0.5775747	8171.696	0.9998685
Uij[7,20]	-0.0021774	0.0048818	0.4498383	-0.8823389	-0.3053032	0.0004164	0.2954453	0.8920188	8490.945	0.9999999
Uij[7,21]	0.2828488	0.0048261	0.4227466	-0.5486289	-0.0020694	0.2827153	0.5625054	1.1015473	7673.025	0.9998777
Uij[7,22]	0.2111218	0.0053773	0.4739329	-0.7270469	-0.1077460	0.2117789	0.5240689	1.1405523	7768.035	0.9999473
Uij[7,23]	-0.4348333	0.0051215	0.4176412	-1.2611891	-0.7182458	-0.4295870	-0.1606398	0.3867412	6649.742	0.9997878
Uij[7,24]	0.1139629	0.0047761	0.4086911	-0.6701003	-0.1686072	0.1173195	0.3945152	0.9236310	7322.104	1.0003798
Uij[7,25]	0.3257470	0.0048997	0.4168573	-0.5012675	0.0457719	0.3306800	0.6062877	1.1562440	7238.422	1.0002241
Uij[7,26]	0.1063559	0.0045177	0.4060904	-0.6927307	-0.1628985	0.1005658	0.3767784	0.9125101	8079.949	0.9998306
Uij[7,27]	0.2417210	0.0047309	0.4304883	-0.6090570	-0.0453813	0.2394792	0.5293926	1.0927608	8280.224	1.0002968
Uij[7,28]	-0.8042228	0.0049591	0.4580414	-1.7158913	-1.1046225	-0.8117571	-0.4960766	0.0870590	8531.002	0.9998705
Uij[7,29]	0.2501043	0.0044318	0.4036534	-0.5421777	-0.0243821	0.2531225	0.5273927	1.0249491	8295.709	0.9999089
Uij[7,30]	0.1830485	0.0049280	0.4130757	-0.6115548	-0.0974593	0.1832852	0.4598386	0.9832646	7026.119	0.9999477
Uij[8,1]	-0.6797147	0.0052015	0.4675453	-1.5951908	-0.9912328	-0.6836537	-0.3690497	0.2368825	8079.467	1.0000451
Uij[8,2]	0.7530444	0.0048968	0.4133600	-0.0688056	0.4750426	0.7585286	1.0313103	1.5524937	7125.768	0.9998760
Uij[8,3]	0.8767076	0.0049982	0.4194564	0.0467125	0.5999791	0.8764537	1.1592419	1.7065229	7042.932	0.9996744
Uij[8,4]	-1.3931510	0.0060144	0.5399612	-2.4761996	-1.7511404	-1.3889283	-1.0223754	-0.3514750	8060.139	0.9997169
Uij[8,5]	1.0293903	0.0045726	0.4264227	0.1928708	0.7396803	1.0308027	1.3155476	1.8587793	8696.731	0.9998040
Uij[8,6]	1.6629825	0.0052373	0.4351741	0.7973662	1.3748512	1.6663331	1.9563794	2.5070325	6904.170	1.0001921
Uij[8,7]	0.6377627	0.0048004	0.4135980	-0.1742209	0.3574055	0.6422804	0.9095237	1.4354207	7423.358	1.0006607
Uij[8,8]	1.1000020	0.0051864	0.4224096	0.2546472	0.8194431	1.1004458	1.3877358	1.9241408	6633.460	1.0002896
Uij[8,9]	0.0173745	0.0049549	0.4526369	-0.8638235	-0.2838410	0.0200288	0.3198568	0.8929469	8345.180	0.9998040
Uij[8,10]	-0.8083359	0.0050599	0.4564115	-1.7030567	-1.1153930	-0.8080988	-0.4964228	0.0680808	8136.347	0.9997124
Uij[8,11]	-0.2558014	0.0050491	0.4605078	-1.1509798	-0.5615123	-0.2536856	0.0486054	0.6454935	8318.568	0.9997518
Uij[8,12]	1.2360673	0.0051452	0.4127775	0.4359177	0.9547951	1.2324319	1.5174841	2.0433249	6436.101	1.0001881
Uij[8,13]	-1.4924469	0.0062529	0.5559413	-2.6339068	-1.8553862	-1.4735235	-1.1110078	-0.4568472	7904.775	0.9999526
Uij[8,14]	-1.2421309	0.0056253	0.5036012	-2.2304790	-1.5726332	-1.2420751	-0.9088601	-0.2481939	8014.580	1.0007184
Uij[8,15]	-0.0076941	0.0053476	0.4471726	-0.8754602	-0.3123613	-0.0002802	0.2946612	0.8626898	6992.616	0.9998336
Uij[8,16]	-1.3804106	0.0059410	0.5642882	-2.5352336	-1.7493490	-1.3675735	-1.0004769	-0.3081336	9021.487	0.9997923
Uij[8,17]	-1.3944819	0.0062225	0.5364612	-2.4952343	-1.7546625	-1.3827286	-1.0274493	-0.3913429	7432.617	1.0001927
Uij[8,18]	-2.1729524	0.0063300	0.5818025	-3.3246618	-2.5606298	-2.1556583	-1.7737270	-1.0943774	8447.919	0.9998806
Uij[8,19]	-1.3740754	0.0058964	0.5164762	-2.4153540	-1.7141067	-1.3672165	-1.0261003	-0.3724640	7672.202	1.0001024
Uij[8,20]	-0.4537292	0.0049502	0.4750314	-1.3906823	-0.7768226	-0.4501857	-0.1315285	0.4734747	9208.560	1.0002069
Uij[8,21]	1.1933456	0.0049073	0.4256895	0.3661585	0.9043413	1.1891295	1.4818523	2.0227081	7524.770	0.9995517
Uij[8,22]	-1.0947476	0.0054924	0.4983996	-2.0852497	-1.4272500	-1.0819927	-0.7559473	-0.1399502	8234.507	0.9995982
Uij[8,23]	1.3536038	0.0050467	0.4211611	0.5327337	1.0705955	1.3657610	1.6351796	2.1770506	6964.331	0.9998251
Uij[8,24]	1.2921909	0.0051902	0.4152882	0.4850221	1.0101123	1.2892075	1.5709785	2.1062766	6402.122	1.0001962
Uij[8,25]	0.4860198	0.0049657	0.4251098	-0.3635677	0.2099469	0.4869041	0.7721711	1.3240835	7328.959	0.9998540
Uij[8,26]	1.0710632	0.0047021	0.4094161	0.2780966	0.7864531	1.0680716	1.3409294	1.8947021	7581.408	1.0002360
Uij[8,27]	-0.7347508	0.0053484	0.4636911	-1.6525125	-1.0457051	-0.7355525	-0.4167170	0.1814844	7516.329	1.0003472
Uij[8,28]	-0.5760778	0.0049569	0.4655035	-1.4891086	-0.8932855	-0.5756722	-0.2580407	0.3286495	8819.219	0.9999394
Uij[8,29]	1.0755167	0.0047433	0.4125635	0.2759515	0.7979133	1.0762203	1.3591285	1.8652500	7565.156	0.9997998
Uij[8,30]	0.5604330	0.0049169	0.4190213	-0.2687412	0.2728515	0.5647204	0.8447049	1.3685136	7262.553	0.9999104
lp__	-3349.4802784	0.4482605	15.6277068	-3380.9917464	-3359.5467733	-3349.4478159	-3338.7833075	-3319.5832264	1215.429	1.0031396

4.4 RQ3 Results and Plots

First let’s get the HPDI interval for the “strength” parameters. Then we will sample the posterior and rank them and present the ranks with their respective posteriors.

hpdi <- get_HPDI_from_stanfit(ranking.fit)

hpdi_algorithm <- hpdi %>% 
      dplyr::filter(str_detect(Parameter, "a_alg\\[")) %>%
      dplyr::mutate(Parameter=algorithms) #Changing to the algorithms labels

p_alg<-ggplot(data=hpdi_algorithm, aes(x=Parameter))+
  geom_pointrange(aes(
    ymin=HPDI.lower, 
    ymax=HPDI.higher, 
    y=Mean))+
  labs(y="Estimate", x="Algorithm", title = "HPDI interval of the strength of the algorithms")+
  coord_flip()
p_alg #+ plot_annotation(title = 'HPDI interval for the algorithms strength')

Computing the ranks

posterior <- rstan::extract(ranking.fit)
a_alg <- as_tibble(posterior$a_alg)
colnames(a_alg) <- algorithms

#sampling from the posterior
s <- dplyr::sample_n(a_alg, size = 1000, replace=T)
s <- dplyr::mutate(s, rown = row_number())
wide_s <- tidyr::pivot_longer(s, cols=all_of(algorithms), names_to = "Algorithm", values_to = "a_alg")
rank_df <- wide_s %>% 
  dplyr::group_by(rown) %>% 
  dplyr::mutate(Rank = rank(-a_alg, ties.method = 'random')) %>% 
  dplyr::ungroup() %>% 
  dplyr::select(-a_alg) %>% 
  dplyr::group_by(Algorithm) %>% 
  dplyr::summarise(MedianRank = median(Rank),
                   VarianceRank = var(Rank)) %>% 
  dplyr::arrange(MedianRank)

Probability of CMAES to beat Random Search and probability of Differential Evolution beating random search

inv_logit <- function(x){
  y<-exp(x)/(1+exp(x))
  return(y)
}

p_cmaes_beat_rs <- as.data.frame(inv_logit(s$CMAES-s$RandomSearch1))
colnames(p_cmaes_beat_rs) <- c('x')
quantile(p_cmaes_beat_rs$x, 0.05)

      5% 
0.497772 

quantile(p_cmaes_beat_rs$x, 0.95)

     95% 
0.819959 

quantile(p_cmaes_beat_rs$x, 0.5)

      50% 
0.6769183 

#raw data
draw <- df_out %>% 
  dplyr::filter(algo0_name=='CMAES' & algo1_name=='RandomSearch1')

(nrow(draw)-sum(draw$y))/nrow(draw) #average of the data

[1] 0.5833333

# 
# p_de_beat_rs <- as.data.frame(inv_logit(s$DifferentialEvolution-s$RandomSearch1))
# colnames(p_de_beat_rs) <- c('x')
# quantile(p_de_beat_rs$x, 0.05)
# quantile(p_de_beat_rs$x, 0.95)
# quantile(p_de_beat_rs$x, 0.5)

we can see that in this case the probability of CMAES beating RS is between 0.50-0.82 with average of 0.67

rank_df_table <- rank_df
colnames(rank_df_table) <- c("Algorithm","Median Rank", "Variance of the Rank")
kable(rank_df_table, "html") %>% 
  kable_styling(bootstrap_options = c('striped',"hover", "condensed" )) %>% 
  kableExtra::scroll_box(width = "100%")

Algorithm	Median Rank	Variance of the Rank
DifferentialEvolution	1	0.1961962
PSO	2	0.3008569
CMAES	3	0.3364004
RandomSearch1	4	0.4719429
RandomSearch2	5	0.5139530
CuckooSearch	6	0.2099339
SimulatedAnnealing	7	0.0069980
NelderMead	8	0.0049800

a_alg <- c("a_alg[1]",
           "a_alg[2]",
           "a_alg[3]",
           "a_alg[4]",
           "a_alg[5]",
           "a_alg[6]",
           "a_alg[7]",
           "a_alg[8]")
rename_pars <- c(paste(rep('a_',length(algorithms)),algorithms, sep = ""),'s')
t<-create_table_model(ranking.fit, pars = c(a_alg, 's'), renamepars =  rename_pars)
colnames(t)<-c("Parameter", "Mean", "HPD low", "HPD high")
saveRDS(t,'./statscomp-paper/tables/datafortables/ranking-par-table.RDS')