diff --git a/README.md b/README.md index 4e10fed..57f264a 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,25 @@ -[![Open in Codespaces](https://classroom.github.com/assets/launch-codespace-7f7980b617ed060a017424585567c406b6ee15c891e84e1186181d67ecf80aa0.svg)](https://classroom.github.com/open-in-codespaces?assignment_repo_id=14492361) -# Autograding Example: Python -This example project is written in Python, and tested with pytest. +[![Open in Codespaces](https://classroom.github.com/assets/launch-codespace-7f7980b617ed060a017424585567c406b6ee15c891e84e1186181d67ecf80aa0.svg)](https://classroom.github.com/open-in-codespaces?assignment_repo_id=14492361) +# Investigation of Multi-Objective Bayesian Optimization of QM9 Dataset -## The assignment -The tests are failing right now because the method isn't outputting the correct string. Fixing this up will make the tests green. +This project will investigate the application of multi-objective Bayesian optimization (specifically EHVI- and parEGO-based methods) to benchmark several multi-objective optimization tasks with QM9 dataset. The objective is to develop specific guidelines about the choice of surrogate and acquisition functions in the context of Multi-Objective Bayesian Optimization for molecular property optimization. -## Setup command +References: -See `postCreateCommand` from [`devcontainer.json`](.devcontainer/devcontainer.json). +Balandat, Maximilian and Karrer, Brian and Jiang, Daniel R. and Daulton, Samuel and Letham, Benjamin and Wilson, Andrew Gordon and Bakshy, Eytan (2020) BoTorch: A Framework for Efficient Monte-Carlo Bayesian Optimization +Raghunathan Ramakrishnan, Pavlo O. Dral, Matthias Rupp & O. Anatole von Lilienfeld (2014) Quantum chemistry structures and properties of 134 kilo molecules -## Run command -`pytest` +# Results & Discussion -## Notes -- pip's install path is not included in the PATH var by default, so without installing via `sudo -H`, pytest would be unaccessible. +In this work, we used the QM9 dataset (that contains about 134K molecules) to demonstrate the power of EHVI-based algorithms. In the interest of time and computational cost, we used randomly selected 1000 structures. We designed a few multi-objective tasks for this project, which are not necessarily motivated by any scientific design goals. But our results show the promise of using BO for similar kind of multi-objective molecular properties optimization. For featurization, we used the coulomb matrix (CM) and smooth overlap of atomic orbitals (SOAP) methods. For GP kernels, we used RBF, and Matern kernel. We compared three different acquisition functions, namely, 1. qEHVI, 2. qNEHVI, and 3. random data acquisition. In all the simulation results provided below, we used 50 initial random seed points. One main limitation of this work is, due to time constraints, we didn't run the simulation several times to have statistically robust results. + +## Maximizing mu (dielectric constant) and alpha (polarizability) + +This problem aims to find materials with a high dielectric constant in conjunction to a high polarizability. Below figure shows the optimization profile along with the pareto fronts for all three acquistion functions. + +![maximization of mu and alpha](experiments/mu_alpha/mu_alpha_results.png) + +# Maximizing mu and minimizing bandgap + +This problem is trying to find the molecules with a high dielectric constant with a low bandgap (high electrical conductivity). + +![Maximization of mu with minimizing gap](experiments/mu_gap/mu_gap_results.png) diff --git a/experiments/mu_alpha/mu_alpha_results.csv b/experiments/mu_alpha/mu_alpha_results.csv new file mode 100644 index 0000000..a7e291f --- /dev/null +++ b/experiments/mu_alpha/mu_alpha_results.csv @@ -0,0 +1,955 @@ +,iteration_qEHVI,iteration_qNEHVI,iteration_random,mu,alpha +0,,,,5.9707,71.88 +1,38.0,,,0.2636,93.18 +2,,,,1.2873,77.08 +3,,,,3.0073,77.91 +4,,,,2.8932,75.17 +5,,,,4.4851,52.58 +6,,45.0,,3.8824,79.11 +7,0.0,0.0,0.0,3.5459,79.68 +8,,,,2.6776,70.34 +9,,,,1.4854,71.62 +10,,,,2.6513,77.82 +11,,,43.0,0.2761,74.53 +12,,,,0.6192,76.21 +13,,,,1.4315,81.82 +14,5.0,,,1.154,88.2 +15,,33.0,,1.4656,100.95 +16,,,,2.2409,68.19 +17,,,,0.5272,77.91 +18,,,,1.2452,68.23 +19,,,,2.6976,75.42 +20,33.0,,,1.3599,88.31 +21,,,,1.5818,80.14 +22,0.0,0.0,0.0,1.2618,81.82 +23,,,,6.5665,56.81 +24,,,,1.9795,76.44 +25,,,,3.0199,79.71 +26,,,,1.8333,69.67 +27,,,18.0,1.9724,84.95 +28,,,,4.0216,73.19 +29,,,,1.7662,71.69 +30,,,,1.9201,77.57 +31,,,,0.7198,84.66 +32,,,,3.9478,81.98 +33,,,,3.4006,77.2 +34,,,,1.9661,83.48 +35,,,,4.5782,73.82 +36,,,,2.3996,88.76 +37,,16.0,,1.3817,88.74 +38,6.0,,,0.0869,91.64 +39,,,,3.3087,66.9 +40,,,,1.4938,72.9 +41,,,,5.7855,61.83 +42,,,,3.6748,74.84 +43,,,,3.0812,74.96 +44,,,,4.2566,80.28 +45,,,,1.6596,72.16 +46,,,39.0,4.4495,59.28 +47,,,,2.9733,76.76 +48,,,,1.8861,70.94 +49,,,,0.6521,78.27 +50,,,,3.9099,76.7 +51,,,,3.6805,73.08 +52,,,,2.5821,69.89 +53,,,,2.4961,78.23 +54,,,,3.408,72.26 +55,0.0,0.0,0.0,1.3426,82.23 +56,,,,2.7683,76.38 +57,11.0,,,1.9357,88.53 +58,,,,2.1968,79.43 +59,,,,1.3762,83.15 +60,,,,2.8986,72.63 +61,,,,2.1997,70.2 +62,,,,1.511,83.31 +63,,,,1.1409,81.28 +64,,,,3.0386,77.08 +65,,,,1.0464,77.29 +66,,,,1.9638,84.36 +67,,,,1.7468,64.44 +68,,,,1.0117,79.17 +69,,,,1.804,75.93 +70,0.0,0.0,0.0,1.1906,68.18 +71,,,,3.2353,49.38 +72,,,,3.809,62.55 +73,16.0,,,2.5207,86.43 +74,,,,1.6646,87.29 +75,,,,2.7923,75.92 +76,,,,2.3373,71.68 +77,,,,1.2697,78.74 +78,,,,1.7676,68.36 +79,0.0,0.0,0.0,2.2843,76.8 +80,,39.0,,4.3603,84.73 +81,,,,2.5166,83.06 +82,,,,1.8912,82.28 +83,,,,1.9723,67.87 +84,,,,1.269,84.91 +85,,,,1.3285,84.56 +86,,,,1.4852,59.34 +87,,,,2.8479,81.39 +88,,,,1.7331,78.78 +89,32.0,,,0.8193,92.05 +90,,,,4.4952,61.37 +91,,,,1.8364,74.81 +92,,,,1.5556,78.5 +93,,,,2.486,71.87 +94,,,,3.1075,73.09 +95,28.0,,,5.7591,80.83 +96,,,,2.2063,81.19 +97,0.0,0.0,0.0,1.6235,72.02 +98,,,,4.8041,60.17 +99,,,,1.5773,72.86 +100,,,,1.5561,78.01 +101,,,,0.4357,78.47 +102,,,,2.4237,54.01 +103,,,,2.825,74.46 +104,,,,4.257,73.64 +105,,,17.0,4.0078,83.86 +106,,,,3.7184,77.07 +107,,,,1.2488,74.85 +108,,,,3.9547,68.58 +109,,,,4.1384,74.52 +110,,,,1.6608,66.92 +111,,,,4.3935,71.14 +112,,,,3.0669,53.52 +113,,,,3.0326,79.11 +114,,,,1.2846,71.36 +115,,,,1.8359,68.3 +116,0.0,0.0,0.0,0.9146,77.02 +117,,,,1.3725,76.65 +118,,,,4.919,73.17 +119,,,,2.5294,73.45 +120,,,,3.6115,82.85 +121,,,,2.8354,74.36 +122,,,,2.6679,77.6 +123,,,12.0,1.9613,85.16 +124,,,,5.2713,73.71 +125,,26.0,,11.1767,86.37 +126,,,,1.9073,79.03 +127,,,,3.3747,76.2 +128,,,,3.5876,69.01 +129,,,,3.659,76.95 +130,,,,4.0166,76.32 +131,,,,1.558,79.82 +132,,,,3.3447,71.92 +133,,,27.0,4.8477,79.46 +134,,,,1.5547,69.7 +135,0.0,0.0,0.0,3.5992,70.7 +136,,,,5.8058,70.26 +137,,,,1.5625,80.7 +138,,,,4.0207,68.83 +139,0.0,0.0,0.0,0.9016,72.53 +140,,,,1.3381,84.93 +141,,27.0,,1.2577,88.9 +142,48.0,,,1.2712,89.54 +143,,,40.0,4.6409,74.76 +144,,,,2.4605,61.73 +145,,,,0.7479,71.36 +146,,,,2.5919,78.24 +147,0.0,0.0,0.0,1.0944,85.53 +148,,,,1.5784,73.19 +149,,,,2.9822,79.08 +150,,,,3.6453,64.56 +151,0.0,0.0,0.0,3.5817,55.65 +152,,,35.0,2.2392,75.22 +153,,,,1.0289,70.38 +154,,,,3.3758,83.52 +155,,7.0,,0.8114,91.74 +156,,,,3.9654,74.8 +157,,,,3.6427,68.38 +158,,,,1.004,84.16 +159,,,,4.6669,67.55 +160,,,,3.026,78.81 +161,,,,4.38,65.63 +162,,,,1.8315,66.79 +163,21.0,,,1.5728,84.69 +164,,,,2.3303,62.83 +165,,,,1.4316,79.77 +166,,,,2.3043,73.43 +167,,,,2.3036,78.25 +168,,,,2.6548,77.3 +169,,,,2.3671,71.21 +170,,,,4.0571,78.01 +171,,,,1.0455,81.89 +172,,,,5.098,76.05 +173,,,,5.942,62.03 +174,,,,2.1513,81.04 +175,,,,6.5098,76.9 +176,,4.0,,3.2603,87.94 +177,,,,1.021,83.07 +178,,,,3.6753,74.1 +179,,,,3.2992,71.12 +180,14.0,,,0.246,91.19 +181,,,,1.3891,77.59 +182,,,,1.3776,72.42 +183,,,,3.9298,68.52 +184,,,,1.7955,75.3 +185,,,,2.9807,74.9 +186,,,,3.7615,81.97 +187,,,,2.0279,76.66 +188,,,,2.3336,71.03 +189,,,,2.6977,74.63 +190,,,,0.2151,70.45 +191,26.0,,,3.2933,79.74 +192,,,,2.5671,70.09 +193,,,,2.0793,49.75 +194,,,,3.5837,69.95 +195,,,,3.083,74.29 +196,,21.0,,1.2219,86.22 +197,,,,1.2725,62.95 +198,,,,4.4788,75.06 +199,,,,3.6203,77.14 +200,,,,0.3824,72.13 +201,,,,7.1271,80.68 +202,,,16.0,2.8382,71.86 +203,,,,1.6463,70.71 +204,,,,1.418,62.84 +205,,,,4.0675,69.46 +206,,,,1.3908,74.72 +207,,,,2.172,74.07 +208,,,,2.6251,77.58 +209,,,,2.8553,82.32 +210,0.0,0.0,0.0,4.1128,75.93 +211,,,,3.2164,66.33 +212,,,,4.9854,75.18 +213,,9.0,,1.3052,91.78 +214,,23.0,,2.6737,81.69 +215,,,,3.873,69.07 +216,,,,1.8963,84.81 +217,,,,1.8833,71.84 +218,,,,6.3962,70.05 +219,,,,3.6282,69.75 +220,,,34.0,3.9195,57.19 +221,,,,2.3241,81.88 +222,,,,1.8503,87.69 +223,,,,1.8419,66.74 +224,,,,2.1715,61.24 +225,,,,3.1648,78.59 +226,,,,3.8794,73.79 +227,,,,2.4291,55.89 +228,,,,2.6368,80.23 +229,,,,4.2039,70.32 +230,,30.0,,3.5399,84.82 +231,,,,2.147,78.58 +232,,,,3.828,63.68 +233,,,,2.8123,76.08 +234,,,,1.0703,74.32 +235,,22.0,,4.011,86.52 +236,,,,4.8474,70.96 +237,,,,4.128,67.52 +238,,,,3.0149,77.8 +239,,,,3.516,73.25 +240,0.0,0.0,0.0,1.5416,73.58 +241,0.0,0.0,0.0,4.5158,78.25 +242,,,,2.4992,80.01 +243,37.0,,,1.8065,89.02 +244,,,,1.6505,64.15 +245,,,,4.3499,77.35 +246,,,,4.4783,77.79 +247,,,,2.7921,85.89 +248,,,,1.871,73.96 +249,,,,2.276,71.07 +250,,,,2.6492,80.44 +251,,,,4.0359,73.34 +252,35.0,,,0.9659,84.6 +253,,,,5.3567,64.59 +254,,,,0.7997,63.78 +255,,,,1.479,84.97 +256,,,,2.4992,78.79 +257,,,,1.9895,83.92 +258,,,,4.9073,72.5 +259,0.0,0.0,0.0,0.1946,89.33 +260,,,,1.7086,81.32 +261,,,,4.0452,73.63 +262,,,,1.8558,69.6 +263,,,,1.8513,71.99 +264,,,,3.6646,73.06 +265,,,,2.2535,73.12 +266,,,,2.4777,69.29 +267,0.0,0.0,0.0,5.1749,71.59 +268,,,,1.6535,72.53 +269,,,,3.0321,69.75 +270,,,,3.2399,69.0 +271,,,,5.165,76.8 +272,,,,4.8811,65.63 +273,,,,2.7418,81.04 +274,,,,2.6986,76.2 +275,,47.0,,1.4744,84.89 +276,,,,5.4607,71.66 +277,,,,3.96,79.55 +278,,,,2.8774,68.75 +279,,,,2.4491,68.46 +280,22.0,,,3.5064,82.64 +281,,49.0,,2.8552,81.78 +282,,,,5.566,77.55 +283,0.0,0.0,0.0,2.1899,81.16 +284,,,,2.4215,57.26 +285,,,,1.7488,68.58 +286,,,,1.2721,77.49 +287,0.0,0.0,0.0,3.7177,60.8 +288,,29.0,,1.0846,90.65 +289,,,,1.7992,82.39 +290,,,,1.6272,80.19 +291,,,,3.6336,73.89 +292,0.0,0.0,0.0,4.831,77.61 +293,,,,5.2836,76.65 +294,,,,1.5778,78.52 +295,,,,3.8469,71.01 +296,,,,1.609,71.17 +297,19.0,,,4.2219,87.22 +298,,,30.0,3.0022,59.11 +299,,,,1.4228,69.9 +300,,,3.0,2.1926,76.98 +301,,,,3.1675,71.98 +302,,,,4.3871,72.36 +303,,,,3.0708,70.62 +304,,,,2.4262,82.74 +305,,,,3.7953,76.21 +306,,,,3.9866,54.36 +307,,,,4.4486,64.99 +308,,,,1.3974,73.09 +309,,,,2.9812,65.54 +310,,5.0,,9.3767,78.89 +311,,,,2.6,58.91 +312,,,,5.9374,70.41 +313,,,,1.6505,80.54 +314,,,,3.5342,63.76 +315,,,,3.1307,71.64 +316,,,,2.3936,73.79 +317,,,,2.408,55.4 +318,,,,2.5798,61.58 +319,,,,5.4285,66.28 +320,,,,3.8078,84.8 +321,,,,2.8849,82.24 +322,,,,3.205,70.1 +323,,,49.0,1.2022,78.92 +324,,,,2.4568,75.33 +325,,,,0.6041,86.51 +326,,,,6.7162,68.24 +327,,,,1.6244,76.46 +328,,,,3.3548,76.93 +329,47.0,,,0.7798,84.16 +330,,,,1.127,84.71 +331,,,,2.6307,70.07 +332,,,,2.6766,51.32 +333,,,,2.7625,64.47 +334,,,,1.5332,81.64 +335,,,,3.4855,64.22 +336,,,6.0,1.9246,60.69 +337,,,,4.0323,69.16 +338,,,,1.6186,72.43 +339,,,,3.3317,73.62 +340,7.0,,,4.9035,69.32 +341,,,,5.9169,68.17 +342,,,,3.9448,76.48 +343,,,2.0,4.6669,79.22 +344,,,,2.6805,72.18 +345,,,,2.9748,61.89 +346,,,,0.5925,77.66 +347,,,,4.6123,69.23 +348,,,,4.5987,75.8 +349,,,,3.9593,66.33 +350,,,,1.3399,85.01 +351,,,,3.165,66.55 +352,,50.0,,3.4764,79.7 +353,,,,5.7142,68.68 +354,,,,1.139,81.77 +355,,,,1.8133,55.96 +356,,,,2.8718,72.24 +357,,,,2.9969,78.02 +358,,,,2.6706,71.53 +359,,,,0.9873,75.21 +360,,,,3.2276,57.77 +361,,,,3.7432,65.59 +362,,,,2.5896,84.79 +363,,,,3.2382,72.21 +364,,,,4.4725,74.64 +365,,,,2.885,77.03 +366,,,42.0,1.296,84.66 +367,,,37.0,1.0255,69.85 +368,,,,1.9021,74.49 +369,,,,3.6755,81.07 +370,,,38.0,5.1,68.93 +371,,,,2.3034,78.12 +372,45.0,,,1.7527,86.67 +373,,,26.0,3.3153,75.2 +374,,,,1.1436,81.41 +375,,,,3.0647,86.0 +376,,,,4.8194,73.97 +377,,,,2.4664,68.55 +378,,,,4.7087,81.55 +379,,,,1.8787,64.9 +380,41.0,,,3.8424,83.36 +381,,,,0.8788,80.93 +382,,,,1.8246,84.18 +383,,,,1.9801,75.06 +384,,,,2.6703,77.54 +385,,,,1.5238,75.95 +386,,,,2.8487,64.28 +387,,,,2.4381,60.01 +388,,18.0,,1.323,86.46 +389,,,,3.4487,68.54 +390,,,,2.5141,81.08 +391,,,,2.2869,69.64 +392,44.0,,,1.6384,85.09 +393,,,,3.9324,74.83 +394,,,,1.7678,72.76 +395,,,,2.1496,70.22 +396,0.0,0.0,0.0,0.1074,95.83 +397,,,8.0,1.5346,79.03 +398,,,,1.929,64.02 +399,,,,3.6038,75.39 +400,,,13.0,3.9623,69.26 +401,,,,1.8971,80.14 +402,,,,2.6196,68.97 +403,,,,4.4768,57.51 +404,,,,3.6026,78.42 +405,3.0,,,0.0585,91.3 +406,,,,1.6942,80.89 +407,,,,2.1082,84.16 +408,20.0,,,0.6042,92.15 +409,,,,1.4814,74.11 +410,,,,4.5933,73.22 +411,,,,2.9229,68.72 +412,,,,4.4014,66.83 +413,,,,1.2128,59.05 +414,,,,3.8176,65.14 +415,,,,2.2489,62.85 +416,0.0,0.0,0.0,1.651,77.83 +417,,1.0,,1.3529,91.7 +418,,,,1.549,74.88 +419,0.0,0.0,0.0,3.6236,82.03 +420,,,,5.1164,79.14 +421,,,,4.8657,79.44 +422,,,,3.2137,64.69 +423,,32.0,,2.8356,81.68 +424,,,,0.2766,69.38 +425,,,,2.1785,69.82 +426,,,,1.7996,82.62 +427,,,,4.4022,64.1 +428,,,,1.4898,73.03 +429,,,,2.2051,56.3 +430,,,,3.9199,79.63 +431,,,,1.7886,80.59 +432,,,,1.7079,70.42 +433,,,,2.1465,84.56 +434,,,,3.4818,71.22 +435,,,,5.9599,56.52 +436,17.0,,,0.8932,93.76 +437,,,,3.7283,59.55 +438,,37.0,,4.244,83.36 +439,,,,3.8301,77.15 +440,,,,2.6852,73.0 +441,,,,1.6656,72.19 +442,,,,4.696,77.99 +443,,,,1.4469,79.57 +444,,,,4.8736,88.12 +445,,,,4.7888,74.16 +446,,,,3.8064,73.82 +447,,,,4.071,76.82 +448,0.0,0.0,0.0,2.0794,83.58 +449,,,,2.8776,75.18 +450,,,,4.3339,72.85 +451,,,,1.5901,83.03 +452,12.0,,,1.0976,91.2 +453,,,,4.4351,63.6 +454,,,,2.495,80.0 +455,,,,2.4439,82.89 +456,,,,4.202,80.04 +457,,,,1.8872,70.96 +458,,,,2.7611,67.47 +459,,,,2.1995,80.38 +460,,36.0,,4.3169,85.04 +461,,,,3.8378,69.69 +462,,,,1.0051,79.15 +463,0.0,0.0,0.0,1.2389,74.35 +464,,,11.0,4.0771,61.57 +465,,34.0,,1.5094,85.13 +466,,,,2.2923,55.73 +467,,,,1.7861,80.67 +468,,,,1.4338,91.21 +469,,,,1.9142,61.49 +470,,,,5.3149,71.28 +471,,,,3.9401,67.33 +472,,,,2.8597,77.62 +473,,,,3.6805,65.58 +474,,,,3.2374,85.44 +475,,,,3.3002,80.68 +476,,,,2.1887,78.59 +477,,,,1.0615,78.73 +478,,,,1.5061,67.25 +479,46.0,,,0.8092,95.58 +480,39.0,,,0.9785,82.37 +481,,,,3.6121,52.8 +482,,,,1.1827,79.06 +483,,,,1.7371,64.4 +484,,,,1.781,84.26 +485,,,,2.1539,72.8 +486,,,9.0,2.7821,62.27 +487,,,,4.0379,71.78 +488,,,32.0,2.3074,69.85 +489,,,,1.5219,79.66 +490,,,,2.94,66.59 +491,,,,4.5127,73.95 +492,,,,4.9466,62.64 +493,,25.0,,2.5432,80.92 +494,,,,1.251,64.02 +495,,,36.0,1.3904,70.39 +496,,,,3.5746,55.64 +497,,,,1.432,84.37 +498,,,,2.9755,79.08 +499,,,,1.0416,82.36 +500,0.0,0.0,0.0,4.2387,85.22 +501,,,,1.3518,67.73 +502,,,,2.0922,67.7 +503,,,,4.2697,58.62 +504,,,,3.4575,69.58 +505,,,,0.5587,88.05 +506,,,,2.2015,66.21 +507,,,,2.3985,81.05 +508,,,,5.1918,81.46 +509,,,,1.2998,78.16 +510,,28.0,,2.9608,81.23 +511,,,,2.3412,66.28 +512,,,,0.7598,67.69 +513,,31.0,,0.2969,89.57 +514,0.0,0.0,0.0,0.8597,82.89 +515,,15.0,,1.2194,85.47 +516,,,,1.4225,79.0 +517,,,,2.2306,78.67 +518,,,,2.7081,70.86 +519,,,,1.733,81.28 +520,0.0,0.0,0.0,1.2231,61.92 +521,,,,1.7584,66.74 +522,,,,3.4748,72.35 +523,,,,2.8301,82.0 +524,,,,1.2253,76.16 +525,31.0,,,0.6509,94.61 +526,,,,3.634,76.03 +527,,,22.0,0.9699,73.46 +528,,3.0,,3.7524,89.92 +529,42.0,,,1.7692,88.55 +530,,,,14.0608,81.62 +531,,,,3.0944,76.21 +532,,,,5.2359,74.33 +533,,,,4.3812,77.74 +534,,,,1.7124,68.17 +535,,,,1.4478,74.54 +536,,,,7.1421,69.77 +537,,,,3.1247,74.73 +538,,,,1.8264,69.79 +539,,,,1.8547,78.58 +540,,,,3.6301,79.79 +541,,,,1.7034,72.03 +542,36.0,,,3.2214,88.28 +543,,,,4.7933,75.67 +544,18.0,,,1.0888,88.0 +545,,,,3.2299,68.26 +546,,,,1.6625,74.65 +547,,,,3.4257,82.02 +548,,,,2.3449,74.84 +549,,,,3.3666,83.99 +550,,,,1.7413,77.75 +551,,42.0,,4.2081,77.59 +552,,,,1.8452,70.66 +553,,,,0.898,71.86 +554,,,,1.7324,74.25 +555,,,,2.718,74.99 +556,13.0,,,1.6272,88.95 +557,,,,1.4112,82.31 +558,,,,4.3468,65.58 +559,,,,4.6763,77.94 +560,,,,3.8275,72.41 +561,,,,3.6081,78.65 +562,,,,2.6238,77.34 +563,,,,0.8515,78.54 +564,,,,3.8414,72.13 +565,,,,3.6489,76.77 +566,0.0,0.0,0.0,1.2552,75.34 +567,,,,3.1285,70.12 +568,0.0,0.0,0.0,1.7984,76.35 +569,,,31.0,2.9467,67.73 +570,,,,4.2174,69.15 +571,,,,2.2138,78.53 +572,,,,3.1679,73.32 +573,,,,5.1817,81.28 +574,,,,1.1678,84.03 +575,,,,3.5541,80.59 +576,,,,2.4287,77.86 +577,10.0,,,0.995,86.27 +578,,,,4.4823,72.67 +579,,,,0.6992,77.18 +580,,,,3.7817,82.93 +581,,,,1.5906,75.59 +582,,,,1.2292,76.19 +583,,,,0.1679,79.55 +584,,,,1.5434,72.49 +585,,,,1.9545,69.49 +586,,,,2.0053,65.41 +587,,,,2.5876,74.45 +588,,,,3.8621,63.19 +589,,,,3.6632,76.2 +590,,,,2.5397,71.03 +591,,,,5.068,77.15 +592,,,,3.3407,76.68 +593,27.0,,,0.2351,88.49 +594,,,,1.3609,81.78 +595,,,,2.9047,62.26 +596,,,,0.5565,65.13 +597,,,,3.1558,68.6 +598,,,21.0,2.4995,81.26 +599,,,,2.4017,59.91 +600,,14.0,,1.425,82.24 +601,,40.0,,2.9703,85.95 +602,,,,2.5632,72.0 +603,,,,3.1585,76.02 +604,,,,4.3519,70.52 +605,,,,2.6379,81.4 +606,,,,4.3141,75.36 +607,,,,5.1825,74.57 +608,,,,2.4859,63.97 +609,,,,5.7223,90.14 +610,,,46.0,0.7567,78.09 +611,,,7.0,3.4946,76.14 +612,0.0,0.0,0.0,3.3904,76.69 +613,,,,3.0777,79.42 +614,,,,3.2896,56.29 +615,,,,1.6894,67.89 +616,,,,1.8932,78.99 +617,,,,1.4579,72.99 +618,,,,1.5017,70.28 +619,,,10.0,2.0862,66.01 +620,,,,3.4933,69.91 +621,,,,1.6105,72.39 +622,,,,2.1113,74.3 +623,,,5.0,1.6712,73.45 +624,0.0,0.0,0.0,2.7428,79.25 +625,,,,3.4953,55.38 +626,,35.0,,3.3733,82.8 +627,,,,2.8542,61.29 +628,,,,5.7462,63.52 +629,,,,2.462,57.94 +630,,,,1.0858,83.94 +631,,,,2.5775,78.11 +632,,,,3.3302,56.72 +633,50.0,,,0.8866,82.34 +634,,,,1.1731,82.03 +635,,,25.0,1.0706,73.73 +636,,,,2.1434,78.14 +637,,,,4.1767,76.87 +638,,,,1.4922,84.33 +639,,,,3.374,65.47 +640,,,15.0,2.0362,76.69 +641,,,,5.0998,74.26 +642,,,,2.7616,71.53 +643,,,,3.4302,79.36 +644,,,,0.4879,76.61 +645,,,,6.0551,63.63 +646,,,,4.0365,67.75 +647,,,,1.637,60.89 +648,,,,2.7712,43.08 +649,,,,2.3613,77.1 +650,,,,1.0414,83.25 +651,,,,2.5379,72.81 +652,,,,2.2167,83.15 +653,,,,3.0191,68.42 +654,,,24.0,4.0902,72.94 +655,,,,2.0682,88.83 +656,,,,3.0559,70.09 +657,,,,3.2199,80.41 +658,,,,0.1133,74.26 +659,,,,1.9667,64.26 +660,,,,2.9406,74.8 +661,0.0,0.0,0.0,0.2448,77.8 +662,,,,3.8469,62.3 +663,,,,0.8488,75.71 +664,,,,2.9301,69.49 +665,,,,1.2835,84.97 +666,,,,3.9183,74.13 +667,,,,1.7031,84.64 +668,,,,4.8076,78.29 +669,,8.0,,2.7709,85.44 +670,,,,1.4775,87.87 +671,,,,5.1537,57.25 +672,,,,2.9602,70.65 +673,,,,4.117,66.34 +674,,,,3.5877,78.64 +675,,,,3.9097,67.8 +676,,,,3.2936,78.26 +677,,,,2.6978,78.65 +678,,,,3.6176,70.63 +679,,,,1.5644,78.8 +680,,,,2.313,75.22 +681,,,,7.0618,67.4 +682,,,1.0,0.818,62.41 +683,,,,3.6542,78.7 +684,0.0,0.0,0.0,6.534,69.93 +685,,,,4.3886,74.22 +686,,,,1.8359,74.34 +687,,,48.0,3.7065,56.39 +688,,,,4.4181,68.2 +689,,,,1.4048,80.07 +690,,,,1.5816,81.34 +691,,,,4.8844,84.97 +692,,,,3.7883,64.12 +693,,,,1.8604,69.88 +694,,,,2.4278,56.55 +695,,,,3.6309,73.38 +696,,13.0,,2.8708,86.71 +697,0.0,0.0,0.0,5.5378,68.09 +698,,,,2.8036,69.97 +699,,,,2.1095,74.41 +700,,,,3.6492,68.08 +701,,20.0,,0.2361,95.02 +702,,,,2.3528,77.79 +703,0.0,0.0,0.0,3.3641,32.83 +704,,,,2.7577,67.08 +705,,,,2.1853,87.87 +706,,,,3.2563,73.45 +707,,,33.0,2.1477,77.32 +708,,,,1.7483,81.01 +709,,,,2.9689,77.68 +710,,,,1.6791,80.09 +711,,38.0,,1.9076,78.48 +712,,,,1.7429,75.96 +713,,,,3.2403,69.79 +714,,,,0.291,85.7 +715,,,,3.248,66.88 +716,8.0,,,0.1587,95.51 +717,,,,2.2604,70.77 +718,,,,2.331,81.46 +719,25.0,,,0.3297,91.06 +720,,,,2.3469,80.24 +721,,,,6.2546,74.5 +722,,,,2.6009,74.08 +723,,,,1.8765,69.94 +724,,41.0,,2.434,84.46 +725,,,,2.6105,68.91 +726,,46.0,,4.0237,81.08 +727,,,,5.4466,75.78 +728,,2.0,,0.0661,95.3 +729,,,,1.5081,78.68 +730,,,,1.225,80.08 +731,,,,1.3996,78.02 +732,40.0,,,0.1246,93.13 +733,,,,2.6659,72.96 +734,0.0,0.0,0.0,0.0584,84.48 +735,,,,2.0601,73.72 +736,,,,1.2643,78.41 +737,,,,1.261,81.65 +738,,,,4.9889,72.52 +739,,,,2.0586,86.33 +740,,24.0,,3.3506,83.01 +741,,,,1.8836,66.63 +742,,,50.0,1.6994,67.79 +743,,12.0,,0.0764,95.73 +744,,,4.0,4.2349,76.68 +745,,43.0,,2.4943,82.69 +746,,,,2.3,78.26 +747,,,,2.5385,71.26 +748,0.0,0.0,0.0,2.0767,72.87 +749,,,,4.7974,71.36 +750,,,,1.5186,70.24 +751,,,,3.1715,82.02 +752,,,,2.137,72.98 +753,,,,3.2491,77.52 +754,,,,1.9691,69.41 +755,,,,6.2828,80.43 +756,,,,2.6739,63.72 +757,,,,1.8814,77.33 +758,,,,3.0933,81.67 +759,30.0,,,4.3122,83.41 +760,,,,1.4818,72.31 +761,,,,1.6129,78.23 +762,,,,1.6077,79.41 +763,,,28.0,3.9438,82.29 +764,,,,0.7892,73.24 +765,,,,1.3778,62.69 +766,,,,1.1691,77.68 +767,,,,4.589,76.12 +768,,,,0.2579,75.93 +769,,,,3.0374,68.01 +770,,,,1.5762,82.24 +771,,,,1.8019,79.48 +772,,,,0.821,81.79 +773,,,44.0,1.5121,80.41 +774,,,,3.1499,79.13 +775,,,23.0,5.7198,51.27 +776,,,,2.1217,75.22 +777,34.0,,,1.5551,89.06 +778,,,,2.5404,71.75 +779,,,,2.6714,64.86 +780,,,,1.2366,73.76 +781,,,,1.4011,68.31 +782,,,,3.667,78.85 +783,,,,3.6358,71.91 +784,,,,5.1252,77.91 +785,,,,2.684,81.97 +786,,,,1.8246,69.99 +787,,,,1.6602,70.97 +788,,,,0.9583,60.93 +789,0.0,0.0,0.0,1.9663,79.47 +790,,,,1.5957,60.8 +791,,,,0.7524,79.7 +792,,,,3.1313,75.33 +793,,,,2.0376,74.33 +794,,,,17.7162,74.31 +795,,,,2.1286,76.4 +796,,,,1.3717,81.86 +797,,,,1.9409,56.47 +798,,,,1.5664,70.55 +799,,,,2.438,76.17 +800,,,,3.7825,72.67 +801,,,,4.5417,68.02 +802,,11.0,,4.2386,81.18 +803,,,,2.0785,71.95 +804,,,,1.5846,69.67 +805,,,,1.1903,75.32 +806,,,,3.8643,71.39 +807,,,,1.3738,80.13 +808,,,,0.996,79.86 +809,,,,1.5501,75.74 +810,,,,1.5016,66.83 +811,43.0,,,2.9907,79.61 +812,,,,6.435,67.8 +813,,,,0.3672,64.55 +814,,,,1.9606,76.9 +815,,,,4.2133,65.34 +816,,,,4.1362,80.03 +817,,,,1.3484,64.04 +818,,,,2.1,83.65 +819,,,,3.7834,73.37 +820,,,,1.9424,76.3 +821,,,,3.7475,80.63 +822,0.0,0.0,0.0,1.4235,66.81 +823,,,,1.731,66.73 +824,2.0,,,2.3845,91.0 +825,,,,1.6507,72.79 +826,,,,1.9409,82.2 +827,0.0,0.0,0.0,2.5373,73.93 +828,,,,2.8058,83.36 +829,,,,4.1778,77.39 +830,,,,2.1842,67.64 +831,15.0,,,1.352,88.93 +832,0.0,0.0,0.0,3.5847,87.02 +833,,,,2.4547,69.51 +834,,,,1.9795,74.28 +835,,17.0,,1.0399,84.9 +836,0.0,0.0,0.0,2.5796,71.73 +837,,,,1.5416,81.34 +838,,,,3.6788,73.77 +839,,,,2.6465,82.14 +840,,,,4.8194,77.61 +841,,,,1.6262,77.52 +842,,,,2.3992,74.12 +843,,,,2.477,80.63 +844,,,,2.4625,60.1 +845,,,29.0,1.1867,74.11 +846,,,,5.5865,58.89 +847,,,,3.2114,73.26 +848,,,,2.5552,56.26 +849,23.0,,,0.2381,97.21 +850,,,,4.1612,76.45 +851,,,,1.5134,67.92 +852,,,,3.3785,76.86 +853,49.0,,,1.2549,88.76 +854,,,,3.2521,81.86 +855,,,,1.3937,71.03 +856,,,,1.7836,75.26 +857,,,,3.2758,80.69 +858,,,,5.0972,72.22 +859,,,14.0,5.0143,83.25 +860,,,,2.5122,62.59 +861,,,41.0,1.6336,64.77 +862,,,,3.9331,67.29 +863,,48.0,,4.3773,78.77 +864,,,,3.2031,82.24 +865,,,,1.9151,65.84 +866,,,,6.2139,71.09 +867,,,,3.7967,71.82 +868,,,,1.8233,68.22 +869,0.0,0.0,0.0,1.0891,68.65 +870,,,,1.1493,71.14 +871,,,,2.5846,75.16 +872,,,,4.0091,70.32 +873,,,,2.3358,65.84 +874,,,,2.3814,82.66 +875,,,20.0,1.1407,73.45 +876,,,,4.3194,77.97 +877,,,,5.1203,65.12 +878,,,,0.9819,73.02 +879,,,,4.3633,60.12 +880,9.0,,,0.1806,89.48 +881,,,,3.9695,70.26 +882,,,,1.6157,69.09 +883,,,,4.1015,61.93 +884,,,,2.1948,79.96 +885,,,,3.8521,76.25 +886,,,,8.8749,52.78 +887,,,,4.9799,74.12 +888,,,,1.5695,83.62 +889,,,,1.7033,76.39 +890,,,,1.7456,79.97 +891,,,,3.6551,65.88 +892,,,,6.5576,78.61 +893,,,19.0,3.6826,79.17 +894,,,,3.2556,68.59 +895,29.0,,,1.5279,83.85 +896,,,,1.6697,82.03 +897,,,,4.74,73.0 +898,0.0,0.0,0.0,0.1508,90.72 +899,,,,1.8714,78.69 +900,0.0,0.0,0.0,4.1499,73.02 +901,,,,5.3679,82.36 +902,0.0,0.0,0.0,3.696,77.55 +903,4.0,,,0.9342,86.55 +904,,,,2.822,68.65 +905,,,,5.6386,66.99 +906,,,45.0,5.2515,76.98 +907,,,,3.9074,67.83 +908,,,,0.8186,80.55 +909,,,,3.9124,70.28 +910,,,,3.819,75.52 +911,,,,1.5733,74.07 +912,,6.0,,6.8054,73.99 +913,1.0,,,2.6234,79.96 +914,0.0,0.0,0.0,2.8611,81.57 +915,,,,2.8367,69.02 +916,,,,1.9001,74.93 +917,,,,1.9283,69.19 +918,,,,2.0153,69.38 +919,,,,3.1413,75.76 +920,,,,2.7987,71.64 +921,,,,5.9931,79.08 +922,,,,2.0892,86.78 +923,,,,3.7451,69.99 +924,,10.0,,5.9914,69.75 +925,,,,5.0913,75.98 +926,24.0,,,1.5992,91.69 +927,,,,6.3854,72.85 +928,,,,4.4962,72.54 +929,,,,2.0351,75.98 +930,,,,1.639,79.79 +931,0.0,0.0,0.0,2.3852,81.58 +932,,,,4.2991,72.32 +933,,,,1.8027,82.77 +934,,,,2.4304,78.31 +935,,,,2.3234,71.31 +936,,,,0.5669,77.8 +937,,,,0.9052,74.78 +938,,44.0,,1.6769,76.6 +939,0.0,0.0,0.0,2.7414,74.89 +940,,,,0.6505,77.32 +941,,,,2.9751,68.86 +942,,,,2.2533,68.15 +943,,,47.0,3.3274,69.77 +944,,,,2.5473,83.64 +945,,,,3.0622,72.69 +946,,,,3.3356,64.34 +947,,,,1.3113,68.68 +948,,,,2.491,74.15 +949,,,,4.299,72.26 +950,,19.0,,1.209,84.99 +951,,,,4.4461,79.79 +952,0.0,0.0,0.0,3.6005,75.25 +953,,,,3.3987,64.79 diff --git a/experiments/mu_alpha/mu_alpha_results.png b/experiments/mu_alpha/mu_alpha_results.png new file mode 100644 index 0000000..173e4cb Binary files /dev/null and b/experiments/mu_alpha/mu_alpha_results.png differ diff --git a/experiments/mu_alpha/script.py b/experiments/mu_alpha/script.py new file mode 100644 index 0000000..01a41e1 --- /dev/null +++ b/experiments/mu_alpha/script.py @@ -0,0 +1,28 @@ +from pathlib import Path +import sys + +import matplotlib.pyplot as plt + +sys.path.append(str(Path(__file__).parents[2])) + +from src.mobo_qm9 import MOBOQM9, MOBOQM9Parameters +from src.utils import plot_results + +params = MOBOQM9Parameters(featurizer="CM", + kernel="Matern", + surrogate_model="GaussianProcess", + targets=["mu", "alpha"], + target_bools=[True, True], + num_total_points=1000, + num_seed_points=50, + n_iters=50, + num_candidates=1) + +moboqm9 = MOBOQM9(params) +moboqm9.run_optimization() +fig = plot_results(moboqm9.dataframe, [True, True]) +fig.tight_layout() +plt.savefig("mu_alpha_results.png") +plt.show() +moboqm9.dataframe.to_csv("mu_alpha_results.csv") + diff --git a/experiments/mu_gap/mu_gap_results.csv b/experiments/mu_gap/mu_gap_results.csv new file mode 100644 index 0000000..9a6f0eb --- /dev/null +++ b/experiments/mu_gap/mu_gap_results.csv @@ -0,0 +1,967 @@ +,iteration_qEHVI,iteration_qNEHVI,iteration_random,mu,gap +0,35.0,,,1.2401,0.2582 +1,0.0,0.0,0.0,2.827,0.2171 +2,36.0,,,4.2454,0.2488 +3,38.0,,,3.4226,0.2187 +4,39.0,,,2.2703,0.2825 +5,40.0,,,0.84,0.2326 +6,42.0,,,1.4939,0.2782 +7,,,27.0,2.561,0.3228 +8,43.0,,,4.9214,0.2756 +9,44.0,,,1.5197,0.2221 +10,45.0,,,0.7222,0.2303 +11,46.0,,,2.7173,0.2053 +12,0.0,0.0,0.0,3.7065,0.2399 +13,47.0,,,2.5307,0.3267 +14,48.0,,,2.2519,0.2193 +15,49.0,,,0.9048,0.2828 +16,50.0,,,2.2713,0.2771 +17,,,,0.6192,0.2665 +18,,,,4.8859,0.2511 +19,,,,3.7766,0.2643 +20,,,,1.3413,0.336 +21,,,,2.8454,0.2817 +22,,,,1.5676,0.2594 +23,,,,1.3263,0.2224 +24,,,,3.117,0.2626 +25,,,,2.2316,0.2388 +26,,,5.0,1.4358,0.1536 +27,,,,1.151,0.2156 +28,,,,6.2963,0.2826 +29,,,,2.6976,0.2987 +30,,,,2.0685,0.2427 +31,,,,3.6086,0.2908 +32,,,,4.2488,0.2283 +33,,,,1.6493,0.2488 +34,,,,2.1165,0.3007 +35,,,,3.9303,0.2785 +36,,,,2.1133,0.1825 +37,,,,2.6389,0.2199 +38,,,,1.4577,0.3141 +39,,,,6.0206,0.265 +40,,,,2.4565,0.2297 +41,,,,1.6902,0.3148 +42,,,,2.3979,0.2301 +43,,,,2.328,0.2198 +44,,,,4.022,0.261 +45,,,,2.2564,0.3116 +46,,,,4.486,0.2558 +47,,,,3.8133,0.2113 +48,,,,4.3031,0.1751 +49,,,,1.7854,0.2624 +50,,,,2.1375,0.2051 +51,,,,1.8653,0.3189 +52,,,,2.2904,0.3253 +53,,,24.0,3.716,0.1832 +54,,,,1.3489,0.25 +55,0.0,0.0,0.0,1.4225,0.1492 +56,,,,2.6718,0.1853 +57,,15.0,,5.0547,0.1923 +58,,,,4.4438,0.2674 +59,,,,3.4132,0.2454 +60,,,1.0,2.3714,0.1739 +61,,,,3.4149,0.2746 +62,,,,5.0219,0.1961 +63,,,,3.9367,0.2781 +64,,,,3.9931,0.2222 +65,,,,4.27,0.2531 +66,,,,3.0065,0.2085 +67,,,,2.5512,0.1917 +68,,35.0,,4.6163,0.1892 +69,,13.0,,6.44,0.1494 +70,,,,3.3124,0.2364 +71,,,,1.2603,0.3239 +72,,,,1.7198,0.2844 +73,,,,3.4365,0.2273 +74,,,,1.6917,0.2994 +75,,,,2.9401,0.3037 +76,,4.0,,5.955,0.1782 +77,0.0,0.0,0.0,2.8986,0.2046 +78,0.0,0.0,0.0,4.5244,0.214 +79,,,,1.0464,0.3164 +80,,,,1.2864,0.2911 +81,,,,2.9422,0.216 +82,,,40.0,1.3803,0.3129 +83,,,8.0,1.8207,0.1968 +84,,,,1.0302,0.307 +85,,,,2.6689,0.2486 +86,0.0,0.0,0.0,4.0473,0.2052 +87,,,,4.5421,0.2286 +88,,,,1.2693,0.3344 +89,0.0,0.0,0.0,1.9978,0.2929 +90,,,,3.415,0.1808 +91,,,,3.0869,0.3105 +92,3.0,,,4.9059,0.1591 +93,15.0,,,4.5556,0.2119 +94,,,,1.5307,0.2165 +95,,,6.0,2.3991,0.2142 +96,,,,1.1769,0.2985 +97,,,,1.2388,0.2496 +98,,,,3.0307,0.2717 +99,,,,3.1469,0.2249 +100,,,,1.2558,0.3056 +101,,,41.0,1.5862,0.2526 +102,,,,2.8545,0.2775 +103,,,,2.8326,0.222 +104,,,,1.0341,0.2619 +105,,,,1.5888,0.205 +106,,,,2.9626,0.312 +107,,,,1.8254,0.301 +108,,,,1.4349,0.2989 +109,,,,1.9595,0.2255 +110,,,,2.7636,0.225 +111,,,,2.7092,0.3144 +112,,,,5.0772,0.2195 +113,,,,1.4378,0.2132 +114,,,,5.1209,0.2725 +115,,,,3.461,0.2148 +116,,,,5.3418,0.218 +117,,,,0.2589,0.2345 +118,,,,2.7881,0.2203 +119,,,,2.1395,0.21 +120,,,,1.1481,0.1614 +121,0.0,0.0,0.0,2.0988,0.2202 +122,,,,2.5149,0.2859 +123,0.0,0.0,0.0,2.742,0.1903 +124,,,,3.465,0.1673 +125,,,,3.6695,0.2201 +126,,,,3.7257,0.2603 +127,,,,3.641,0.1998 +128,,,,3.5914,0.2384 +129,,,,1.3819,0.3239 +130,,,,2.1907,0.2907 +131,,,,2.6309,0.21 +132,,,,5.4483,0.2519 +133,26.0,,,5.8404,0.1878 +134,,,,0.2397,0.3161 +135,,,,2.1312,0.2432 +136,,,,3.5868,0.2861 +137,,,,4.3658,0.2508 +138,,,,4.1723,0.261 +139,,,,2.4901,0.2151 +140,,,,4.3271,0.2382 +141,,,,3.9184,0.2346 +142,,,,1.5869,0.3215 +143,,,,8.957,0.172 +144,,45.0,,6.0153,0.2052 +145,,,,0.1469,0.3405 +146,,,,1.2517,0.2645 +147,,,,2.1992,0.2469 +148,,,,2.7125,0.2126 +149,,,,2.7951,0.2448 +150,,,,2.6664,0.246 +151,,,,1.8339,0.237 +152,,,,2.6734,0.3216 +153,,,,0.7594,0.228 +154,,,,1.9763,0.283 +155,,,,2.5751,0.2804 +156,,31.0,,5.2389,0.2704 +157,,,,1.8417,0.2829 +158,,,,3.965,0.2797 +159,,,,4.3623,0.2411 +160,,,,1.1107,0.2789 +161,,,,3.3123,0.2283 +162,,,,4.1988,0.2703 +163,,,,1.5728,0.3226 +164,,,,3.5477,0.2437 +165,,,,2.6141,0.2056 +166,,,,1.445,0.2112 +167,,,,2.604,0.3075 +168,,,21.0,4.6281,0.2567 +169,0.0,0.0,0.0,2.4369,0.219 +170,37.0,,,5.2518,0.2372 +171,,,,1.8926,0.3149 +172,,,,2.9269,0.1561 +173,,,,1.5509,0.3185 +174,,,,5.2746,0.2662 +175,,,,3.568,0.2634 +176,,,,3.8632,0.2822 +177,,,,1.1106,0.2364 +178,,,,1.6651,0.253 +179,,,,0.088,0.3375 +180,,,,3.2994,0.2655 +181,,40.0,,3.6712,0.2761 +182,,,,3.3578,0.2948 +183,,,,0.9178,0.2224 +184,,,,3.1022,0.1858 +185,,,,2.213,0.2144 +186,20.0,,,1.4827,0.1567 +187,,,,1.6512,0.3201 +188,,,12.0,1.9609,0.2416 +189,,,,0.1104,0.2984 +190,,,,1.8627,0.2422 +191,,,2.0,2.5056,0.2182 +192,,27.0,,2.6669,0.1501 +193,,,34.0,2.0927,0.1948 +194,,47.0,,1.8777,0.1562 +195,,3.0,,1.9222,0.1667 +196,,,47.0,2.2343,0.2141 +197,,,,4.2237,0.2368 +198,,,,1.7662,0.2225 +199,2.0,,,3.9609,0.2153 +200,,,,2.0267,0.2119 +201,,,,0.9174,0.3054 +202,,,,3.8907,0.2105 +203,11.0,,,5.7535,0.265 +204,0.0,0.0,0.0,5.1835,0.2607 +205,,,,2.595,0.2674 +206,,,,8.7867,0.2577 +207,,,,1.7196,0.2494 +208,,,,2.8558,0.2268 +209,,,,1.4504,0.3158 +210,,,,3.4836,0.3082 +211,,34.0,,4.1334,0.268 +212,,,,1.2332,0.3164 +213,,,,4.4383,0.2505 +214,,,,1.3999,0.2762 +215,,,,4.1871,0.1938 +216,,,,1.3909,0.3204 +217,,,,2.6514,0.2226 +218,,,,1.1838,0.2587 +219,,,,6.3962,0.2547 +220,,,,1.4042,0.3116 +221,,,,2.8869,0.29 +222,,,35.0,3.9195,0.276 +223,,,,3.6225,0.2999 +224,10.0,,,4.3423,0.1641 +225,,,,4.1705,0.2481 +226,,,,1.9367,0.2731 +227,32.0,,,3.2007,0.1727 +228,,,,2.4789,0.218 +229,0.0,0.0,0.0,4.2904,0.2012 +230,,,,1.9301,0.2868 +231,,,,4.0875,0.1989 +232,,,,2.1045,0.1873 +233,,,,2.035,0.2771 +234,,,,1.5953,0.2218 +235,,,,5.0326,0.2859 +236,,29.0,,4.7953,0.2515 +237,5.0,,,4.1682,0.2259 +238,,,,2.6592,0.3004 +239,,,,1.9712,0.2448 +240,,16.0,,1.9685,0.1978 +241,,,,1.0232,0.3288 +242,,,,2.2241,0.2049 +243,,,,2.6074,0.2753 +244,,,,3.9652,0.2313 +245,,,,0.4381,0.1634 +246,,48.0,,2.5109,0.2525 +247,,,,1.7369,0.3221 +248,,,,1.4345,0.3183 +249,,8.0,,6.5855,0.1831 +250,13.0,,,3.3966,0.1586 +251,,,,0.1451,0.3352 +252,,,,2.703,0.3072 +253,,,,1.6687,0.3109 +254,,,,1.4575,0.2342 +255,,,,1.2885,0.2912 +256,,,50.0,3.7609,0.266 +257,,,,3.4924,0.2435 +258,,,,0.9486,0.3275 +259,,,,2.7626,0.2128 +260,,,,1.8956,0.2941 +261,,,,4.3026,0.263 +262,,,,3.3213,0.3022 +263,,,,1.1434,0.2381 +264,,,,2.2715,0.3245 +265,,,,2.9878,0.2415 +266,,,,1.7499,0.3114 +267,,,,1.0893,0.2494 +268,,,,0.254,0.2477 +269,,,,1.8635,0.3082 +270,,,,1.7324,0.2428 +271,,,,1.7289,0.3407 +272,,,,3.4084,0.2645 +273,,,3.0,4.9598,0.2052 +274,,18.0,,4.879,0.2377 +275,,,,1.5507,0.3143 +276,,,,1.2521,0.2303 +277,,,,4.0787,0.2144 +278,,,,3.3992,0.2711 +279,,,,1.777,0.221 +280,,,,4.3263,0.2888 +281,,,,1.4766,0.218 +282,,,,3.8436,0.2305 +283,,26.0,,3.482,0.2035 +284,,9.0,,2.4237,0.1618 +285,0.0,0.0,0.0,1.6901,0.2435 +286,0.0,0.0,0.0,1.9489,0.2371 +287,,,,3.0146,0.2557 +288,,,,3.2426,0.2153 +289,,,,3.7933,0.267 +290,,,,3.0395,0.2022 +291,,,,2.5745,0.2416 +292,,,,2.9604,0.2546 +293,27.0,,,3.2886,0.1442 +294,,,,0.0428,0.3891 +295,34.0,,,4.7107,0.1723 +296,,,,1.1647,0.3284 +297,,,,2.1926,0.3206 +298,,,,2.9666,0.2316 +299,0.0,0.0,0.0,4.739,0.3291 +300,,,,1.61,0.3133 +301,,,,1.7544,0.2792 +302,,,,0.675,0.1966 +303,,,,2.2478,0.2353 +304,,,,4.7423,0.2234 +305,,,,3.76,0.2138 +306,,,,1.3438,0.2412 +307,,,,3.1398,0.2839 +308,,,28.0,1.4238,0.2019 +309,,,,1.0181,0.23 +310,,,,1.7326,0.3288 +311,,,,4.794,0.3387 +312,,6.0,,6.7017,0.2411 +313,,,,3.5225,0.1817 +314,,,,2.14,0.2194 +315,,,,3.639,0.234 +316,0.0,0.0,0.0,3.5449,0.219 +317,,,,2.677,0.1546 +318,,,,1.9174,0.2067 +319,,,,4.124,0.2172 +320,,,,2.6995,0.2786 +321,,,,2.2604,0.2554 +322,,,,2.3386,0.2208 +323,,,,2.2726,0.202 +324,0.0,0.0,0.0,3.556,0.2842 +325,,,,1.4047,0.3108 +326,,,,3.8677,0.2628 +327,,,,2.7302,0.2263 +328,,,,3.9925,0.2573 +329,,,,6.154,0.1746 +330,,,,3.9211,0.2132 +331,,,,2.6471,0.2674 +332,,,,2.289,0.2161 +333,,,,0.8538,0.287 +334,,,,2.011,0.3102 +335,,,,2.9846,0.2306 +336,,,,2.2213,0.2636 +337,,,,2.9203,0.2277 +338,,,,4.665,0.2016 +339,,,,1.5347,0.2825 +340,,,,2.3301,0.1913 +341,,,,5.5275,0.2819 +342,,,44.0,2.4942,0.2681 +343,,,,3.4435,0.1987 +344,,17.0,,2.2702,0.3569 +345,,,,1.8827,0.3272 +346,,,,1.3652,0.2333 +347,,,,3.4392,0.2398 +348,,,,2.45,0.3163 +349,,,,1.4143,0.2863 +350,,,,4.738,0.2328 +351,0.0,0.0,0.0,2.0357,0.2374 +352,,,,3.2557,0.2149 +353,,,,1.6502,0.2083 +354,,,,0.3741,0.2065 +355,,,,3.1815,0.164 +356,,,,3.652,0.2647 +357,,,,1.2141,0.3066 +358,,,,3.117,0.3126 +359,,,,4.297,0.2478 +360,,,,1.4818,0.3162 +361,,,,2.2721,0.2581 +362,,,,1.4967,0.2568 +363,,,,1.1108,0.3133 +364,,,,0.2583,0.2393 +365,,,,1.647,0.2872 +366,,,,2.4381,0.2233 +367,,,,2.6815,0.3069 +368,,,,3.7586,0.2781 +369,,,,1.5519,0.3101 +370,,,,1.6926,0.3296 +371,,,,2.7734,0.2512 +372,,,,3.8605,0.2329 +373,,,,5.1753,0.2358 +374,,,,3.8983,0.2004 +375,,,,2.7853,0.1481 +376,,,,3.0851,0.2754 +377,,,,2.4242,0.2187 +378,,,,2.1548,0.233 +379,,,,2.9545,0.2588 +380,,,,1.7243,0.2375 +381,,1.0,,1.7174,0.2374 +382,,,,4.6978,0.1999 +383,,,,1.0064,0.2864 +384,,,,0.3161,0.318 +385,,,,2.5063,0.2457 +386,,,,1.5906,0.2645 +387,0.0,0.0,0.0,2.2968,0.2136 +388,,,,2.39,0.2795 +389,,,,2.5546,0.2766 +390,,,,1.4804,0.3347 +391,,,29.0,1.5562,0.3174 +392,,,7.0,1.6186,0.2684 +393,,,,1.0126,0.2932 +394,,,,3.399,0.239 +395,,,,3.2855,0.2559 +396,,,,2.3901,0.2342 +397,,,,2.0092,0.295 +398,,,,4.5672,0.2234 +399,33.0,,,3.3678,0.1598 +400,,,,2.5469,0.2046 +401,,,,1.5468,0.2919 +402,,,,1.9449,0.2466 +403,,7.0,,3.5068,0.2353 +404,,,,1.1033,0.1941 +405,,,,0.6022,0.2962 +406,,,,2.865,0.2541 +407,,,,2.7153,0.2264 +408,,,,0.5981,0.2977 +409,,,,3.1617,0.1884 +410,,,,3.5,0.2719 +411,,,,2.7229,0.334 +412,,,,1.82,0.3287 +413,,,,2.3061,0.2424 +414,,,,5.4446,0.1821 +415,,,,2.4149,0.22 +416,,,,2.8426,0.2281 +417,,,,3.2165,0.2147 +418,,,,2.7139,0.2182 +419,18.0,,,3.986,0.1788 +420,23.0,,,2.2277,0.2677 +421,,,,2.1287,0.2379 +422,,,,4.548,0.2331 +423,,,,2.2494,0.3142 +424,,,,1.7908,0.2441 +425,,,,1.8443,0.24 +426,,,,5.7422,0.2662 +427,,,,0.1609,0.1479 +428,,,,4.6812,0.2682 +429,,44.0,,3.0929,0.1593 +430,,,,2.6725,0.3136 +431,,,,2.3529,0.1656 +432,,,,3.3778,0.1723 +433,,,,3.2981,0.2135 +434,9.0,,,2.747,0.2013 +435,,,,3.3543,0.2443 +436,,,,3.2082,0.2846 +437,,,,0.6626,0.3106 +438,,,,4.6794,0.2545 +439,,,,1.943,0.2555 +440,,,,0.8393,0.2301 +441,,,,1.7596,0.2597 +442,,,,1.7923,0.2868 +443,,,,4.0111,0.2622 +444,,,,3.2957,0.2598 +445,,,,2.4438,0.183 +446,,,,2.6703,0.2049 +447,,19.0,,4.0331,0.1901 +448,,,,3.367,0.2736 +449,,,,1.9645,0.2407 +450,,,,4.3657,0.2465 +451,,,,1.769,0.2912 +452,,,,1.7211,0.2872 +453,0.0,0.0,0.0,5.7013,0.1937 +454,,,,1.9802,0.2695 +455,,,,3.8593,0.1905 +456,,,,2.8059,0.2621 +457,,,,4.4599,0.2402 +458,,,,1.0062,0.1533 +459,,41.0,,4.9268,0.1589 +460,,,,3.2552,0.2781 +461,,,,2.8604,0.2903 +462,,,,6.441,0.2396 +463,,,,3.5197,0.259 +464,,,,5.3649,0.2668 +465,,,17.0,2.5813,0.2977 +466,,,,1.7779,0.3199 +467,,,10.0,4.9076,0.266 +468,,,,2.9996,0.2168 +469,,42.0,,4.3085,0.2162 +470,,,,1.9135,0.2887 +471,0.0,0.0,0.0,1.3389,0.2991 +472,,,,4.2358,0.229 +473,,,,3.8034,0.2555 +474,,,,2.4638,0.3276 +475,,,14.0,1.5665,0.2545 +476,,,,1.9171,0.2974 +477,,,,3.7958,0.2025 +478,,,,3.18,0.2174 +479,,,,1.4853,0.2596 +480,,,,3.86,0.2534 +481,,,,2.0714,0.2115 +482,,,,3.402,0.294 +483,,,,0.2792,0.2257 +484,,,22.0,2.5693,0.2493 +485,,,,3.2147,0.2314 +486,,,,3.929,0.2978 +487,,,,1.4645,0.2218 +488,,,,1.7774,0.2104 +489,,,,3.0505,0.2104 +490,,,,1.2875,0.3331 +491,,,,1.8348,0.3214 +492,,,,2.1934,0.2363 +493,,,,3.1777,0.143 +494,,,,1.1066,0.2197 +495,,,,1.7106,0.2038 +496,,,,2.2227,0.3057 +497,,,,0.7933,0.2901 +498,,,,4.3831,0.2703 +499,28.0,,,3.5705,0.1587 +500,,,,0.2851,0.2567 +501,,,,1.6874,0.3183 +502,,,,4.3712,0.205 +503,,,,1.2653,0.285 +504,,,,2.6723,0.2186 +505,,,,4.1609,0.2746 +506,,,,2.0357,0.2699 +507,,,,5.9077,0.2842 +508,,,,0.5636,0.1798 +509,,,,1.8032,0.2305 +510,0.0,0.0,0.0,1.9924,0.2502 +511,,,,1.7527,0.2781 +512,,,,2.7522,0.3072 +513,,,,2.2207,0.296 +514,,,,4.2236,0.2609 +515,,32.0,,3.4851,0.1659 +516,,,,1.2111,0.2304 +517,,,,5.0923,0.2044 +518,,,,2.0086,0.2249 +519,,,,2.4938,0.3213 +520,,,,3.219,0.2711 +521,,,,1.8271,0.302 +522,,,,3.5525,0.2615 +523,0.0,0.0,0.0,3.6741,0.2213 +524,,,,2.9065,0.1886 +525,,12.0,,3.1487,0.1777 +526,30.0,,,2.2015,0.2022 +527,,22.0,,3.4302,0.1382 +528,,,,2.0412,0.2974 +529,,,,1.727,0.1721 +530,,,,2.5515,0.3037 +531,,,,2.007,0.2508 +532,,,,1.3418,0.206 +533,,,,1.0312,0.324 +534,0.0,0.0,0.0,2.8763,0.1594 +535,19.0,,,2.9587,0.2065 +536,29.0,,,1.977,0.1739 +537,,,,1.6354,0.3061 +538,,,,2.8354,0.2271 +539,,,25.0,5.6923,0.1859 +540,,,,4.7064,0.2338 +541,,,,1.6749,0.3331 +542,,,,1.3796,0.2493 +543,,,,1.5235,0.2602 +544,,,,1.608,0.2457 +545,,,,3.0513,0.2237 +546,,,,2.7586,0.2082 +547,,,,4.4387,0.2619 +548,,,,1.3051,0.2297 +549,,,,4.0935,0.3041 +550,,,,1.4129,0.2958 +551,,,9.0,2.4158,0.227 +552,,,,3.2929,0.2927 +553,,,,1.9934,0.1855 +554,,,,0.7619,0.2555 +555,,,,2.388,0.2338 +556,,,,1.1734,0.2939 +557,,,,2.5356,0.2077 +558,,,39.0,3.4228,0.2436 +559,,,,2.6321,0.2074 +560,,,,3.6529,0.1927 +561,,,,1.8566,0.2184 +562,,,,4.1496,0.2134 +563,,,,2.1199,0.2466 +564,,,,1.501,0.2888 +565,,,,3.2138,0.2085 +566,,,,1.4996,0.2183 +567,,,,2.3473,0.316 +568,,,,1.3083,0.233 +569,,,,1.4534,0.2315 +570,,,,2.8306,0.2528 +571,,,,1.7683,0.2169 +572,,,,3.0793,0.2575 +573,0.0,0.0,0.0,3.3473,0.216 +574,,,,1.6643,0.3101 +575,,,,1.9765,0.2684 +576,,,,2.0846,0.2909 +577,,,,5.3027,0.2693 +578,,,4.0,1.5423,0.2244 +579,,,,1.5644,0.2417 +580,,,,3.8084,0.2489 +581,,,,1.2094,0.1904 +582,,,23.0,4.1193,0.2422 +583,,,,2.3021,0.3196 +584,,,,2.6169,0.2596 +585,,,,1.4186,0.2108 +586,,,,2.9018,0.1975 +587,0.0,0.0,0.0,2.9842,0.2364 +588,,,,1.5676,0.2831 +589,0.0,0.0,0.0,1.8787,0.2557 +590,,,,2.7412,0.2138 +591,,,,3.7464,0.2451 +592,,38.0,,3.2276,0.1901 +593,,,,1.7498,0.3115 +594,,,,1.8155,0.205 +595,,,,3.9689,0.2463 +596,,,,4.6712,0.1982 +597,,,,1.9094,0.3027 +598,,,11.0,1.1392,0.205 +599,,,,1.046,0.2405 +600,,,,0.5825,0.1969 +601,,,,1.1572,0.3237 +602,,,,3.2468,0.2271 +603,,,,1.8842,0.2209 +604,,,,5.5201,0.2282 +605,,,,0.9542,0.3457 +606,,,,1.9989,0.3182 +607,,,,3.0378,0.2 +608,,,,0.7941,0.1982 +609,,,,3.5729,0.2712 +610,0.0,0.0,0.0,1.1987,0.2455 +611,0.0,0.0,0.0,3.7215,0.2633 +612,,,,2.278,0.2465 +613,,,48.0,4.0386,0.2766 +614,,,,1.5926,0.3256 +615,,,,0.1836,0.3232 +616,,,,4.8964,0.251 +617,,,,3.8943,0.2565 +618,0.0,0.0,0.0,3.0326,0.2223 +619,,,,3.8761,0.2392 +620,,43.0,,1.4168,0.2581 +621,22.0,,,3.452,0.1985 +622,,,,1.3472,0.2124 +623,,,,1.8666,0.157 +624,,,,3.3422,0.1452 +625,,,,0.5445,0.2209 +626,,,,2.0559,0.201 +627,,,,1.7105,0.2904 +628,,,,2.4633,0.3092 +629,0.0,0.0,0.0,1.9721,0.3201 +630,,,13.0,3.1397,0.2505 +631,,,,0.3246,0.2253 +632,,,,0.4462,0.2409 +633,,,,2.2593,0.2727 +634,,,,1.4492,0.3174 +635,,,,4.3284,0.3094 +636,,,,1.5423,0.1648 +637,,,,3.6846,0.2122 +638,,,,1.06,0.2338 +639,,37.0,,3.0353,0.2746 +640,,25.0,,0.4663,0.1584 +641,,,,1.3782,0.3285 +642,,,,2.6074,0.2496 +643,,,45.0,1.6472,0.2609 +644,,36.0,,1.6609,0.1717 +645,,,,2.3994,0.2889 +646,,24.0,,4.9793,0.2532 +647,,,,1.6405,0.2049 +648,,,,1.5568,0.2794 +649,,,,1.6137,0.2807 +650,7.0,,,1.7108,0.2036 +651,,,,3.9064,0.259 +652,0.0,0.0,0.0,1.608,0.2223 +653,,,,3.5037,0.2217 +654,,,,4.0349,0.246 +655,,,,0.2728,0.2577 +656,,,,0.9584,0.2357 +657,,33.0,,3.0983,0.1809 +658,,,,1.6581,0.1992 +659,,,,1.8335,0.2286 +660,,21.0,,3.3219,0.1741 +661,,,,5.3403,0.2263 +662,,,,0.4543,0.3251 +663,,,,3.2131,0.2735 +664,,,,2.4109,0.2132 +665,0.0,0.0,0.0,2.3885,0.215 +666,41.0,,,16.5398,0.1138 +667,,,,1.2953,0.341 +668,,,,1.7344,0.2245 +669,,,,0.8418,0.3115 +670,,,,0.5413,0.1927 +671,,,,1.1034,0.2417 +672,,,,4.3246,0.3361 +673,,,,4.0655,0.2502 +674,,,,2.5595,0.2042 +675,,,,1.6417,0.2218 +676,,,,1.9717,0.1814 +677,,20.0,,2.1261,0.1302 +678,,,,1.5442,0.2031 +679,,,,1.4173,0.2408 +680,,28.0,,4.1664,0.2173 +681,,,,4.17,0.2611 +682,,,,1.5366,0.3275 +683,,,,2.7269,0.2406 +684,,14.0,,5.3249,0.2779 +685,,,,4.2366,0.1661 +686,,,42.0,1.6037,0.2325 +687,0.0,0.0,0.0,2.5866,0.1982 +688,,,,1.5683,0.2606 +689,,,,5.4911,0.2656 +690,,,,2.0421,0.2501 +691,,,,2.2877,0.1911 +692,,,,0.791,0.2503 +693,,2.0,,3.4829,0.1512 +694,,,,1.7508,0.2879 +695,,,,1.4214,0.3027 +696,,,,1.0735,0.2405 +697,,,,1.9486,0.2708 +698,,,,1.3993,0.3121 +699,,,,3.4066,0.2506 +700,,,,3.0008,0.2666 +701,,,,3.5749,0.2874 +702,,,18.0,2.4525,0.2472 +703,,,,0.7491,0.3002 +704,,,,3.6707,0.2553 +705,,,,1.2697,0.1823 +706,,,,1.6113,0.1592 +707,,,20.0,2.2963,0.315 +708,,,,3.4434,0.2307 +709,,,,4.1845,0.3309 +710,,,,2.8168,0.3014 +711,,,,3.4515,0.2626 +712,,,,3.4547,0.2603 +713,,,,3.81,0.3064 +714,,,,1.9742,0.2073 +715,,,,2.7996,0.1854 +716,,,,0.8939,0.2383 +717,,,,2.9746,0.2547 +718,,,,2.1112,0.2378 +719,,,,1.2311,0.3445 +720,,,,4.5443,0.2056 +721,,,,2.5608,0.2507 +722,,,,0.0269,0.3111 +723,,,,5.4232,0.2716 +724,,,,0.7719,0.3087 +725,,,,1.5754,0.3053 +726,,,,1.987,0.2498 +727,,,,3.3194,0.2957 +728,,,,0.9868,0.3265 +729,,,32.0,1.7407,0.2317 +730,,,,3.2968,0.2306 +731,17.0,,,6.5668,0.1597 +732,,,,5.3524,0.2203 +733,,,,1.6841,0.3085 +734,,,30.0,1.8356,0.2373 +735,0.0,0.0,0.0,5.916,0.204 +736,,,,0.499,0.3151 +737,0.0,0.0,0.0,1.4088,0.2471 +738,,,,2.6109,0.3075 +739,,,,1.2036,0.3168 +740,,,,2.7918,0.1688 +741,,,,2.6443,0.2491 +742,,,,3.9016,0.2745 +743,12.0,,,5.5158,0.2076 +744,,,,3.2304,0.2257 +745,31.0,,,3.7721,0.1738 +746,1.0,,,2.4519,0.2282 +747,,,,3.3693,0.2702 +748,,,,1.4392,0.3317 +749,0.0,0.0,0.0,1.6892,0.3159 +750,,,,1.5395,0.2555 +751,,,,3.8603,0.2851 +752,,,,2.9563,0.2049 +753,,,16.0,0.9859,0.209 +754,,,,5.1655,0.2686 +755,,,,3.2791,0.246 +756,,,,1.9391,0.1888 +757,,50.0,,3.0311,0.2252 +758,,,,1.4705,0.2971 +759,,,,1.9378,0.2818 +760,,,,5.5765,0.2307 +761,0.0,0.0,0.0,2.0888,0.2707 +762,,,,1.2553,0.3131 +763,,,,0.7007,0.292 +764,,,,2.4699,0.2797 +765,,,,0.4711,0.2455 +766,,,,1.3018,0.2434 +767,,,,2.6177,0.3288 +768,,,,5.0802,0.2412 +769,,,,5.8411,0.2844 +770,,,,3.7529,0.2364 +771,,,,2.519,0.1991 +772,0.0,0.0,0.0,4.2747,0.2407 +773,,,,5.3965,0.2631 +774,,,,1.629,0.3222 +775,,,,1.7976,0.3088 +776,,,,1.2299,0.2959 +777,,,,1.1661,0.1601 +778,,,,1.9915,0.3199 +779,,,,2.5863,0.313 +780,,,,0.6674,0.1731 +781,0.0,0.0,0.0,2.0579,0.2128 +782,,,,2.8454,0.2242 +783,,,,4.1325,0.2867 +784,,,,4.001,0.2535 +785,,,,3.6959,0.2765 +786,,,,5.4761,0.2807 +787,0.0,0.0,0.0,2.8711,0.2606 +788,,,,0.2539,0.3001 +789,,,,1.2093,0.2898 +790,,,,1.1239,0.2623 +791,,,,0.9855,0.3368 +792,0.0,0.0,0.0,6.0902,0.2817 +793,,,,0.1825,0.3156 +794,,,,6.2675,0.206 +795,,,,2.9847,0.1831 +796,,,,6.0218,0.1921 +797,,,,1.6317,0.312 +798,,46.0,,14.0654,0.1551 +799,,,,0.9807,0.2417 +800,,,,5.8541,0.2171 +801,,,,1.9635,0.3096 +802,,,,1.1174,0.328 +803,,,,3.743,0.1959 +804,,,,0.9927,0.2812 +805,,,,2.413,0.2808 +806,,,,2.0866,0.2415 +807,,,,3.4962,0.164 +808,,,,1.0909,0.2513 +809,,,,2.6131,0.3128 +810,,,,4.1649,0.2355 +811,,,,3.3602,0.3094 +812,,,,2.3104,0.3039 +813,,,,2.9713,0.2902 +814,,,,3.733,0.1679 +815,,,,4.6419,0.2209 +816,,,,1.4305,0.2288 +817,,,,3.1252,0.2698 +818,,,,1.8461,0.2852 +819,,49.0,,5.7171,0.1879 +820,,,,1.7993,0.2343 +821,,,,5.0163,0.2848 +822,,,,2.4844,0.216 +823,,,,0.6525,0.327 +824,,,,0.778,0.2384 +825,,,36.0,3.1666,0.2857 +826,,,,3.8875,0.2316 +827,,,,2.407,0.2544 +828,,,,1.4723,0.3215 +829,,,,1.352,0.3349 +830,,,38.0,2.9045,0.2061 +831,,,,2.8873,0.1766 +832,,,,5.3384,0.2757 +833,,,,1.1424,0.2875 +834,,,,2.401,0.3318 +835,,,,3.3574,0.2887 +836,,,,4.5353,0.2813 +837,,,,1.9303,0.2767 +838,,,,1.4964,0.3058 +839,,,,0.5577,0.3106 +840,,,,1.0856,0.2647 +841,,,,3.8045,0.2729 +842,,,,3.5967,0.1919 +843,0.0,0.0,0.0,1.9532,0.2785 +844,8.0,,,3.2113,0.1555 +845,,,,3.2146,0.2224 +846,,,,2.0175,0.1675 +847,,,,2.7427,0.2478 +848,,,,1.7768,0.2193 +849,24.0,,,1.7909,0.1987 +850,25.0,,,2.9784,0.1514 +851,,,,2.548,0.2187 +852,,,,2.6746,0.264 +853,,,,1.645,0.2171 +854,,,,3.6797,0.2699 +855,,,,1.6619,0.2451 +856,,,,1.8949,0.2449 +857,,,43.0,2.6961,0.2071 +858,,,,0.1028,0.3306 +859,,,,4.3143,0.2558 +860,,,,2.2089,0.1992 +861,,,,0.8792,0.2332 +862,,,,3.6174,0.1801 +863,0.0,0.0,0.0,0.7672,0.34 +864,,,,3.3087,0.1795 +865,,,,3.9181,0.2159 +866,,,,5.7184,0.2804 +867,,,,1.5963,0.309 +868,,,,4.7081,0.208 +869,,5.0,,5.5044,0.2283 +870,,,,1.8429,0.2484 +871,,,,3.9596,0.3247 +872,16.0,,,1.249,0.1064 +873,,,,5.591,0.235 +874,0.0,0.0,0.0,5.9805,0.3269 +875,,,,1.7482,0.2424 +876,,,,4.5514,0.2199 +877,,11.0,,2.5704,0.1481 +878,,,,1.5496,0.3202 +879,,,,2.1997,0.2605 +880,,10.0,,1.1328,0.2206 +881,,,,0.0004,0.3296 +882,,23.0,,5.3473,0.1937 +883,,,,1.9501,0.2618 +884,,,,3.9389,0.2254 +885,,,,2.3843,0.2778 +886,0.0,0.0,0.0,0.7078,0.3182 +887,,,,1.5444,0.2861 +888,,,46.0,1.8767,0.2563 +889,,,,3.6602,0.2632 +890,,,,1.6691,0.233 +891,,,26.0,3.9123,0.223 +892,,39.0,,1.3831,0.1466 +893,,,,6.1302,0.2497 +894,,,,3.1561,0.2865 +895,,,,4.3721,0.1771 +896,,,,1.623,0.2308 +897,,,,2.6578,0.2025 +898,,,,2.4008,0.2186 +899,,,,5.3004,0.3064 +900,,,,5.0507,0.2586 +901,,,,2.7082,0.2802 +902,,,,5.2934,0.2632 +903,,,,2.5145,0.2702 +904,,,,3.4141,0.2081 +905,,,,1.9105,0.2036 +906,,,,1.533,0.2959 +907,,,,1.7915,0.2475 +908,,,,3.6009,0.1919 +909,0.0,0.0,0.0,2.3667,0.2305 +910,,,,1.7576,0.2527 +911,,,,1.8825,0.3501 +912,14.0,,,4.5469,0.325 +913,,,,2.9126,0.2119 +914,21.0,,,5.0021,0.2106 +915,,,,0.2001,0.2616 +916,,,,1.09,0.1995 +917,,,,0.7004,0.3098 +918,,,49.0,2.696,0.208 +919,,,15.0,3.4348,0.219 +920,,,,0.0787,0.3654 +921,,,,2.2543,0.2553 +922,,,33.0,1.802,0.2092 +923,0.0,0.0,0.0,2.2147,0.2808 +924,,,,0.6022,0.1425 +925,,,,3.8873,0.3045 +926,,,,1.7606,0.2322 +927,4.0,,,3.12,0.1774 +928,0.0,0.0,0.0,1.119,0.2876 +929,,,,2.1297,0.3028 +930,,,,1.1491,0.3034 +931,,,,2.0993,0.284 +932,,,,3.6257,0.2395 +933,,,,1.5291,0.2431 +934,,,,0.2535,0.3111 +935,,,,2.0351,0.2429 +936,,,,1.7108,0.301 +937,,,,1.3493,0.2341 +938,,,,2.0934,0.2622 +939,,,,5.0503,0.2876 +940,,,,1.7109,0.2482 +941,,,,2.242,0.2699 +942,,30.0,,4.4539,0.2108 +943,,,,3.6006,0.2061 +944,,,,2.5467,0.3172 +945,,,31.0,3.7893,0.2489 +946,,,,3.9717,0.2907 +947,,,,1.6416,0.2722 +948,,,19.0,4.3041,0.2758 +949,,,,3.5267,0.3276 +950,,,,0.8959,0.3356 +951,,,,1.5768,0.3136 +952,,,,0.2648,0.3022 +953,,,,1.8387,0.2775 +954,,,,1.1481,0.26 +955,,,,1.9757,0.2167 +956,,,,1.7173,0.3117 +957,,,,2.2955,0.2461 +958,,,37.0,3.4383,0.2516 +959,,,,3.3527,0.2271 +960,,,,4.1391,0.2431 +961,,,,1.351,0.3194 +962,,,,1.8111,0.2291 +963,,,,2.2282,0.2776 +964,6.0,,,0.7382,0.1623 +965,,,,2.2243,0.2267 diff --git a/experiments/mu_gap/mu_gap_results.png b/experiments/mu_gap/mu_gap_results.png new file mode 100644 index 0000000..362c285 Binary files /dev/null and b/experiments/mu_gap/mu_gap_results.png differ diff --git a/experiments/mu_gap/script.py b/experiments/mu_gap/script.py new file mode 100644 index 0000000..15c4ab7 --- /dev/null +++ b/experiments/mu_gap/script.py @@ -0,0 +1,28 @@ +from pathlib import Path +import sys + +import matplotlib.pyplot as plt + +sys.path.append(str(Path(__file__).parents[2])) + +from src.mobo_qm9 import MOBOQM9, MOBOQM9Parameters +from src.utils import plot_results + +params = MOBOQM9Parameters(featurizer="SOAP", + kernel="Matern", + surrogate_model="GaussianProcess", + targets=["mu", "gap"], + target_bools=[True, False], + num_total_points=1000, + num_seed_points=50, + n_iters=50, + num_candidates=1) + +moboqm9 = MOBOQM9(params) +moboqm9.run_optimization() +fig = plot_results(moboqm9.dataframe, [True, True]) +fig.tight_layout() +plt.savefig("mu_gap_results.png") +plt.show() +moboqm9.dataframe.to_csv("mu_gap_results.csv") + diff --git a/src/data/soap_featurizer.py b/src/data/soap_featurizer.py index ffcdc51..fb7baba 100644 --- a/src/data/soap_featurizer.py +++ b/src/data/soap_featurizer.py @@ -39,6 +39,10 @@ def get_soap(indices, targets): r_cut = 3.0, n_max = 4, l_max = 3, + periodic=False, + sparse=False, + average="inner", + rbf="gto", ) atoms_list, computed_targets = [], [] diff --git a/src/mobo_qm9.py b/src/mobo_qm9.py index 7b05ad6..4796f61 100644 --- a/src/mobo_qm9.py +++ b/src/mobo_qm9.py @@ -5,13 +5,15 @@ from botorch.utils.multi_objective.box_decompositions import DominatedPartitioning from botorch.models import ModelListGP, SingleTaskGP import gpytorch -from gpytorch.kernels import RBFKernel, MaternKernel +from gpytorch.kernels import RBFKernel from botorch.fit import fit_gpytorch_mll from botorch.models.transforms.input import Normalize from botorch.models.transforms.outcome import Standardize +from gpytorch.mlls.sum_marginal_log_likelihood import SumMarginalLogLikelihood import pandas as pd from .data.cm_featurizer import get_coulomb_matrix +from.data.soap_featurizer import get_soap from .acquisition_functions import optimize_qEHVI, optimize_qNEHVI N_TOTAL_POINTS = 138_728 @@ -32,7 +34,7 @@ class MOBOQM9Parameters(NamedTuple): num_seed_points: Number of seed points to use. Default is 100. n_iters: Number of iterations. Default is 20. """ - featurizer: Literal["ECFP", "CM", "ACSF"] + featurizer: Literal["ECFP", "CM", "SOAP"] kernel: Literal["RBF", "Matern"] surrogate_model: Literal["GaussianProcess", "RandomForest"] targets: List[str] @@ -95,6 +97,9 @@ def get_features_and_targets(self): if self.params.featurizer == "CM": return get_coulomb_matrix(self.total_indices, self.params.targets) + elif self.params.featurizer == "SOAP": + return get_soap(self.total_indices, + self.params.targets) else: raise NotImplementedError @@ -117,7 +122,10 @@ def get_surrogate_model(self, acq): if self.params.kernel == 'RBF': kernel = RBFKernel() elif self.params.kernel == 'Matern': - kernel = MaternKernel() + kernel = gpytorch.kernels.ScaleKernel( + gpytorch.kernels.MaternKernel() + ) + gpytorch.kernels.ScaleKernel( + gpytorch.kernels.LinearKernel()) else: raise ValueError("Unsupported kernel type. Supported types are 'RBF', and 'Matern'.") @@ -132,7 +140,7 @@ def get_surrogate_model(self, acq): for i in range(targets.shape[1])] model = ModelListGP(*models) - mll = gpytorch.mlls.SumMarginalLogLikelihood(model.likelihood, model) + mll = SumMarginalLogLikelihood(model.likelihood, model) fit_gpytorch_mll(mll) return model @@ -256,7 +264,7 @@ def validate_params(self): args: params: Parameters for the MOBOQM9 model. """ - assert self.params.featurizer in ["ECFP", "CM", "ACSF"], "Featurizer must be one of ECFP, CM, or ACSF." + assert self.params.featurizer in ["ECFP", "CM", "SOAP"], "Featurizer must be one of ECFP, CM, or SOAP." assert self.params.kernel in ["RBF", "Matern", "Tanimoto"], "Kernel must be one of RBF, Matern." assert self.params.surrogate_model in ["GaussianProcess", "RandomForest"], "Surrogate model must be one of GaussianProcess, or RandomForest." assert len(self.params.targets) == len(self.params.target_bools), "Number of targets must equal number of target booleans." diff --git a/src/utils.py b/src/utils.py index 52bea09..cc8cdfc 100644 --- a/src/utils.py +++ b/src/utils.py @@ -21,11 +21,11 @@ def plot_results(df, target_bool): ax = ax.flatten() targets = [col for col in df.columns if not col.startswith("iteration")] global_values = torch.tensor(df[targets].values) - global_pareto_idx = is_non_dominated(global_values) - global_pareto_front = global_values[global_pareto_idx] for mask in target_bool: if not mask: global_values[:, mask] *= -1 + global_pareto_idx = is_non_dominated(global_values) + global_pareto_front = global_values[global_pareto_idx] ref_point = global_values.min(0)[0] hv = DominatedPartitioning(ref_point=ref_point, Y=global_values) global_hv = hv.compute_hypervolume().item() @@ -54,6 +54,9 @@ def plot_results(df, target_bool): max_iterations = int(df['iteration_qEHVI'].max()) current_data = df[df['iteration_qEHVI'] <= max_iterations] qEHVI_values = torch.tensor(current_data[targets].values) + for mask in target_bool: + if not mask: + qEHVI_values[:, mask] *= -1 qEHVI_pareto_idx = is_non_dominated(qEHVI_values) qEHVI_pareto_front = qEHVI_values[qEHVI_pareto_idx] ax[1].scatter(qEHVI_pareto_front[:, 0], qEHVI_pareto_front[:, 1], @@ -70,6 +73,9 @@ def plot_results(df, target_bool): max_iterations = int(df['iteration_qNEHVI'].max()) current_data = df[df['iteration_qNEHVI'] <= max_iterations] qNEHVI_values = torch.tensor(current_data[targets].values) + for mask in target_bool: + if not mask: + qNEHVI_values[:, mask] *= -1 qNEHVI_pareto_idx = is_non_dominated(qNEHVI_values) qNEHVI_pareto_front = qNEHVI_values[qNEHVI_pareto_idx] ax[2].scatter(qNEHVI_pareto_front[:, 0], qNEHVI_pareto_front[:, 1], @@ -86,6 +92,9 @@ def plot_results(df, target_bool): max_iterations = int(df['iteration_random'].max()) current_data = df[df['iteration_random'] <= max_iterations] random_values = torch.tensor(current_data[targets].values) + for mask in target_bool: + if not mask: + random_values[:, mask] *= -1 random_pareto_idx = is_non_dominated(random_values) random_pareto_front = random_values[random_pareto_idx] ax[3].scatter(random_pareto_front[:, 0], random_pareto_front[:, 1],